From 3d8d9efd702b08c78e48694feff7792f4fead3e0 Mon Sep 17 00:00:00 2001 From: Millian Lamiaux Date: Thu, 26 Mar 2026 10:47:05 +0100 Subject: [PATCH] feat: YouTube music download system with admin dashboard MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Sidecar architecture: edge functions (auth + DB) → youtube-worker container (youtubei.js for playlist metadata, yt-dlp for audio downloads). - Edge functions: youtube-playlist, youtube-process, youtube-status (CRUD) - youtube-worker sidecar: Node.js + yt-dlp, downloads M4A to Supabase Storage - Admin music page: import playlists, process queue, inline genre editing - 12 music genres with per-track assignment and import-time defaults - DB migrations: download_jobs, download_items tables with genre column - Deploy script and CI workflow for edge functions + sidecar --- .github/workflows/ci.yml | 29 + admin-web/app/music/page.tsx | 640 ++++++++++++++++++ admin-web/components/sidebar.tsx | 2 + admin-web/lib/supabase.ts | 63 ++ admin-web/lib/use-youtube-download.ts | 301 ++++++++ scripts/deploy-functions.sh | 55 ++ supabase/functions/_shared/auth.ts | 62 ++ supabase/functions/_shared/cors.ts | 13 + supabase/functions/_shared/supabase-client.ts | 19 + supabase/functions/_shared/youtube-client.ts | 88 +++ supabase/functions/deno.json | 8 + supabase/functions/main/index.ts | 95 +++ supabase/functions/youtube-playlist/index.ts | 124 ++++ supabase/functions/youtube-process/index.ts | 200 ++++++ supabase/functions/youtube-status/index.ts | 171 +++++ supabase/migrations/002_download_jobs.sql | 96 +++ supabase/migrations/003_music_genre.sql | 18 + youtube-worker/Dockerfile | 21 + youtube-worker/package-lock.json | 190 ++++++ youtube-worker/package.json | 13 + youtube-worker/server.js | 218 ++++++ 21 files changed, 2426 insertions(+) create mode 100644 admin-web/app/music/page.tsx create mode 100644 admin-web/lib/use-youtube-download.ts create mode 100755 scripts/deploy-functions.sh create mode 100644 supabase/functions/_shared/auth.ts create mode 100644 supabase/functions/_shared/cors.ts create mode 100644 supabase/functions/_shared/supabase-client.ts create mode 100644 supabase/functions/_shared/youtube-client.ts create mode 100644 supabase/functions/deno.json create mode 100644 supabase/functions/main/index.ts create mode 100644 supabase/functions/youtube-playlist/index.ts create mode 100644 supabase/functions/youtube-process/index.ts create mode 100644 supabase/functions/youtube-status/index.ts create mode 100644 supabase/migrations/002_download_jobs.sql create mode 100644 supabase/migrations/003_music_genre.sql create mode 100644 youtube-worker/Dockerfile create mode 100644 youtube-worker/package-lock.json create mode 100644 youtube-worker/package.json create mode 100644 youtube-worker/server.js diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7b0be51..50698b8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -196,3 +196,32 @@ jobs: - name: Export web build run: npx expo export --platform web continue-on-error: true + + deploy-functions: + name: Deploy Edge Functions + runs-on: ubuntu-latest + needs: [typecheck, lint, test] + if: github.ref == 'refs/heads/main' && github.event_name == 'push' + steps: + - uses: actions/checkout@v4 + + - name: Deploy to self-hosted Supabase + env: + DEPLOY_HOST: ${{ secrets.SUPABASE_DEPLOY_HOST }} + DEPLOY_USER: ${{ secrets.SUPABASE_DEPLOY_USER }} + DEPLOY_PATH: ${{ secrets.SUPABASE_DEPLOY_PATH }} + run: | + mkdir -p ~/.ssh + echo "${{ secrets.SUPABASE_SSH_KEY }}" > ~/.ssh/deploy_key + chmod 600 ~/.ssh/deploy_key + ssh-keyscan -H $DEPLOY_HOST >> ~/.ssh/known_hosts 2>/dev/null + + rsync -avz --delete \ + --exclude='node_modules' \ + --exclude='.DS_Store' \ + -e "ssh -i ~/.ssh/deploy_key" \ + supabase/functions/ \ + "$DEPLOY_USER@$DEPLOY_HOST:$DEPLOY_PATH/" + + ssh -i ~/.ssh/deploy_key "$DEPLOY_USER@$DEPLOY_HOST" \ + "docker restart supabase-edge-functions" diff --git a/admin-web/app/music/page.tsx b/admin-web/app/music/page.tsx new file mode 100644 index 0000000..5f45b60 --- /dev/null +++ b/admin-web/app/music/page.tsx @@ -0,0 +1,640 @@ +"use client"; + +import { useState, useEffect } from "react"; +import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; +import { Button } from "@/components/ui/button"; +import { Input } from "@/components/ui/input"; +import { Badge } from "@/components/ui/badge"; +import { + Table, + TableBody, + TableCell, + TableHead, + TableHeader, + TableRow, +} from "@/components/ui/table"; +import { + Music, + Download, + Square, + Play, + Pause, + Loader2, + ExternalLink, + CheckCircle2, + XCircle, + Clock, + RefreshCw, + Trash2, +} from "lucide-react"; +import { toast } from "sonner"; +import { + useYouTubeDownload, + type JobWithItems, +} from "@/lib/use-youtube-download"; +import { + Dialog, + DialogContent, + DialogDescription, + DialogFooter, + DialogHeader, + DialogTitle, +} from "@/components/ui/dialog"; +import { Select } from "@/components/ui/select"; +import type { Database, MusicGenre } from "@/lib/supabase"; +import { MUSIC_GENRES, GENRE_LABELS } from "@/lib/supabase"; + +type DownloadJob = Database["public"]["Tables"]["download_jobs"]["Row"]; +type DownloadItem = Database["public"]["Tables"]["download_items"]["Row"]; + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +function formatDuration(seconds: number | null): string { + if (!seconds) return "--:--"; + const m = Math.floor(seconds / 60); + const s = seconds % 60; + return `${m}:${s.toString().padStart(2, "0")}`; +} + +function statusBadge(status: string) { + switch (status) { + case "completed": + return ( + + + Completed + + ); + case "downloading": + case "processing": + return ( + + + {status === "downloading" ? "Downloading" : "Processing"} + + ); + case "failed": + return ( + + + Failed + + ); + default: + return ( + + + Pending + + ); + } +} + +function progressPercent(job: DownloadJob): number { + if (job.total_items === 0) return 0; + return Math.round( + ((job.completed_items + job.failed_items) / job.total_items) * 100 + ); +} + +const genreOptions = MUSIC_GENRES.map((g) => ({ + value: g, + label: GENRE_LABELS[g], +})); + +const genreSelectOptions = [ + { value: "", label: "No genre" }, + ...genreOptions, +]; + +function genreBadge(genre: MusicGenre | null) { + if (!genre) return null; + return ( + + {GENRE_LABELS[genre]} + + ); +} + +// --------------------------------------------------------------------------- +// Sub-components +// --------------------------------------------------------------------------- + +function ImportSection({ + onImport, + isImporting, +}: { + onImport: (url: string, genre?: MusicGenre) => void; + isImporting: boolean; +}) { + const [url, setUrl] = useState(""); + const [genre, setGenre] = useState(""); + + const handleSubmit = (e: React.FormEvent) => { + e.preventDefault(); + const trimmed = url.trim(); + if (!trimmed) return; + onImport(trimmed, (genre || undefined) as MusicGenre | undefined); + setUrl(""); + }; + + return ( + + + + + Import YouTube Playlist + + + +
+ setUrl(e.target.value)} + placeholder="https://www.youtube.com/playlist?list=..." + className="bg-neutral-950 border-neutral-700 text-white placeholder:text-neutral-500 flex-1" + disabled={isImporting} + /> + + onGenreChange(item.id, (val || null) as MusicGenre | null) + } + options={genreSelectOptions} + placeholder="--" + className="w-32" + /> + + + {formatDuration(item.duration_seconds)} + + {statusBadge(item.status)} + +
+ {item.status === "completed" && item.public_url && ( + + )} + + + +
+ {item.status === "failed" && item.error_message && ( +

+ {item.error_message} +

+ )} + + + ))} + + + ); +} + +// --------------------------------------------------------------------------- +// Main page +// --------------------------------------------------------------------------- + +export default function MusicPage() { + const { + jobs, + activeJob, + isProcessing, + isImporting, + fetchJobs, + refreshStatus, + importPlaylist, + startProcessing, + stopProcessing, + deleteJob, + updateItemGenre, + } = useYouTubeDownload(); + + const [selectedJobId, setSelectedJobId] = useState(null); + const [deleteTarget, setDeleteTarget] = useState(null); + const [isDeleting, setIsDeleting] = useState(false); + + // Load jobs on mount + useEffect(() => { + fetchJobs().catch((err) => + toast.error("Failed to load jobs: " + err.message) + ); + }, [fetchJobs]); + + const handleImport = async (url: string, genre?: MusicGenre) => { + try { + const result = await importPlaylist(url, genre); + toast.success( + `Imported "${result.playlistTitle}" — ${result.totalItems} tracks` + ); + setSelectedJobId(result.jobId); + } catch (err) { + toast.error( + "Import failed: " + + (err instanceof Error ? err.message : "Unknown error") + ); + } + }; + + const handleStart = async (jobId: string) => { + setSelectedJobId(jobId); + try { + await startProcessing(jobId); + toast.success("Download complete!"); + } catch (err) { + toast.error( + "Processing error: " + + (err instanceof Error ? err.message : "Unknown error") + ); + } + }; + + const handleSelectJob = async (jobId: string) => { + setSelectedJobId(jobId); + try { + await refreshStatus(jobId); + } catch (err) { + toast.error("Failed to load job details"); + } + }; + + const handleDelete = async () => { + if (!deleteTarget) return; + setIsDeleting(true); + try { + await deleteJob(deleteTarget.id); + toast.success( + `Deleted "${deleteTarget.playlist_title || "Untitled Playlist"}"` + ); + if (selectedJobId === deleteTarget.id) { + setSelectedJobId(null); + } + } catch (err) { + toast.error( + "Delete failed: " + + (err instanceof Error ? err.message : "Unknown error") + ); + } finally { + setIsDeleting(false); + setDeleteTarget(null); + } + }; + + const handleGenreChange = async (itemId: string, genre: MusicGenre | null) => { + try { + await updateItemGenre(itemId, genre); + } catch (err) { + toast.error( + "Failed to update genre: " + + (err instanceof Error ? err.message : "Unknown error") + ); + } + }; + + return ( +
+ {/* Header */} +
+
+

Music

+

+ Download audio from YouTube playlists for workout tracks +

+
+ +
+ + {/* Import section */} +
+ +
+ + {/* Jobs + Detail layout */} +
+ {/* Jobs list */} +
+

+ Jobs ({jobs.length}) +

+ {jobs.length === 0 ? ( +

+ No jobs yet. Import a playlist to get started. +

+ ) : ( + jobs.map((job) => ( + handleSelectJob(job.id)} + onStart={() => handleStart(job.id)} + onStop={stopProcessing} + onDelete={() => setDeleteTarget(job)} + /> + )) + )} +
+ + {/* Detail panel */} +
+ {activeJob ? ( + + + + {activeJob.playlist_title || "Untitled Playlist"} + + + + + + + ) : ( + + + +

+ Select a job to view its tracks +

+ + + )} +
+
+ + {/* Delete confirmation dialog */} + { if (!open) setDeleteTarget(null); }}> + + + Delete Job + + This will permanently delete{" "} + + {deleteTarget?.playlist_title || "this job"} + {" "} + and remove {deleteTarget?.completed_items ?? 0} audio files from + storage. This action cannot be undone. + + + + + + + + +
+ ); +} diff --git a/admin-web/components/sidebar.tsx b/admin-web/components/sidebar.tsx index af4f3e0..1c88158 100644 --- a/admin-web/components/sidebar.tsx +++ b/admin-web/components/sidebar.tsx @@ -10,6 +10,7 @@ import { Users, FolderOpen, ImageIcon, + Music, LogOut, Flame, } from "lucide-react"; @@ -20,6 +21,7 @@ const navItems = [ { href: "/trainers", label: "Trainers", icon: Users }, { href: "/collections", label: "Collections", icon: FolderOpen }, { href: "/media", label: "Media", icon: ImageIcon }, + { href: "/music", label: "Music", icon: Music }, ]; export function Sidebar() { diff --git a/admin-web/lib/supabase.ts b/admin-web/lib/supabase.ts index a2701c7..b2a80cf 100644 --- a/admin-web/lib/supabase.ts +++ b/admin-web/lib/supabase.ts @@ -34,6 +34,28 @@ export type Json = | { [key: string]: Json | undefined } | Json[] +export const MUSIC_GENRES = [ + 'edm', 'hip-hop', 'pop', 'rock', 'latin', 'house', + 'drum-and-bass', 'dubstep', 'r-and-b', 'country', 'metal', 'ambient', +] as const + +export type MusicGenre = typeof MUSIC_GENRES[number] + +export const GENRE_LABELS: Record = { + 'edm': 'EDM', + 'hip-hop': 'Hip Hop', + 'pop': 'Pop', + 'rock': 'Rock', + 'latin': 'Latin', + 'house': 'House', + 'drum-and-bass': 'Drum & Bass', + 'dubstep': 'Dubstep', + 'r-and-b': 'R&B', + 'country': 'Country', + 'metal': 'Metal', + 'ambient': 'Ambient', +} + export interface Database { public: { Tables: { @@ -137,6 +159,47 @@ export interface Database { last_login: string | null } } + download_jobs: { + Row: { + id: string + playlist_url: string + playlist_title: string | null + status: 'pending' | 'processing' | 'completed' | 'failed' + total_items: number + completed_items: number + failed_items: number + created_by: string + created_at: string + updated_at: string + } + Insert: { + id?: string + playlist_url: string + playlist_title?: string | null + status?: 'pending' | 'processing' | 'completed' | 'failed' + total_items?: number + completed_items?: number + failed_items?: number + created_by: string + } + Update: Partial> + } + download_items: { + Row: { + id: string + job_id: string + video_id: string + title: string | null + duration_seconds: number | null + thumbnail_url: string | null + status: 'pending' | 'downloading' | 'completed' | 'failed' + storage_path: string | null + public_url: string | null + error_message: string | null + genre: MusicGenre | null + created_at: string + } + } } } } \ No newline at end of file diff --git a/admin-web/lib/use-youtube-download.ts b/admin-web/lib/use-youtube-download.ts new file mode 100644 index 0000000..104b54f --- /dev/null +++ b/admin-web/lib/use-youtube-download.ts @@ -0,0 +1,301 @@ +"use client"; + +import { useState, useCallback, useRef } from "react"; +import { supabase } from "@/lib/supabase"; +import type { Database, MusicGenre } from "@/lib/supabase"; + +type DownloadJob = Database["public"]["Tables"]["download_jobs"]["Row"]; +type DownloadItem = Database["public"]["Tables"]["download_items"]["Row"]; + +export interface JobWithItems extends DownloadJob { + items: DownloadItem[]; +} + +const PROCESS_DELAY_MS = 1000; + +/** + * Construct a GET request to a Supabase edge function with query params. + * supabase.functions.invoke() doesn't support query params, so we use fetch. + */ +async function invokeGet( + functionName: string, + params?: Record +): Promise { + const { + data: { session }, + } = await supabase.auth.getSession(); + if (!session) throw new Error("Not authenticated"); + + const supabaseUrl = + process.env.NEXT_PUBLIC_SUPABASE_URL || + process.env.EXPO_PUBLIC_SUPABASE_URL || + "http://localhost:54321"; + + const url = new URL(`${supabaseUrl}/functions/v1/${functionName}`); + if (params) { + Object.entries(params).forEach(([k, v]) => url.searchParams.set(k, v)); + } + + const res = await fetch(url.toString(), { + method: "GET", + headers: { + Authorization: `Bearer ${session.access_token}`, + "Content-Type": "application/json", + }, + }); + + if (!res.ok) { + const body = await res.json().catch(() => ({ error: res.statusText })); + throw new Error(body.error || `HTTP ${res.status}`); + } + + return res.json(); +} + +/** + * Send a DELETE request to a Supabase edge function with a JSON body. + */ +async function invokeDelete( + functionName: string, + body: Record +): Promise { + const { + data: { session }, + } = await supabase.auth.getSession(); + if (!session) throw new Error("Not authenticated"); + + const supabaseUrl = + process.env.NEXT_PUBLIC_SUPABASE_URL || + process.env.EXPO_PUBLIC_SUPABASE_URL || + "http://localhost:54321"; + + const res = await fetch(`${supabaseUrl}/functions/v1/${functionName}`, { + method: "DELETE", + headers: { + Authorization: `Bearer ${session.access_token}`, + "Content-Type": "application/json", + }, + body: JSON.stringify(body), + }); + + if (!res.ok) { + const data = await res.json().catch(() => ({ error: res.statusText })); + throw new Error(data.error || `HTTP ${res.status}`); + } + + return res.json(); +} + +/** + * Send a PATCH request to a Supabase edge function with a JSON body. + */ +async function invokePatch( + functionName: string, + body: Record +): Promise { + const { + data: { session }, + } = await supabase.auth.getSession(); + if (!session) throw new Error("Not authenticated"); + + const supabaseUrl = + process.env.NEXT_PUBLIC_SUPABASE_URL || + process.env.EXPO_PUBLIC_SUPABASE_URL || + "http://localhost:54321"; + + const res = await fetch(`${supabaseUrl}/functions/v1/${functionName}`, { + method: "PATCH", + headers: { + Authorization: `Bearer ${session.access_token}`, + "Content-Type": "application/json", + }, + body: JSON.stringify(body), + }); + + if (!res.ok) { + const data = await res.json().catch(() => ({ error: res.statusText })); + throw new Error(data.error || `HTTP ${res.status}`); + } + + return res.json(); +} + +export function useYouTubeDownload() { + const [jobs, setJobs] = useState([]); + const [activeJob, setActiveJob] = useState(null); + const [isProcessing, setIsProcessing] = useState(false); + const [isImporting, setIsImporting] = useState(false); + const abortRef = useRef(null); + + /** Fetch all jobs (list view). */ + const fetchJobs = useCallback(async () => { + const data = await invokeGet<{ jobs: DownloadJob[] }>("youtube-status"); + setJobs(data.jobs); + return data.jobs; + }, []); + + /** Fetch a single job with its items. */ + const refreshStatus = useCallback(async (jobId: string) => { + const data = await invokeGet<{ job: DownloadJob; items: DownloadItem[] }>( + "youtube-status", + { jobId } + ); + const jobWithItems: JobWithItems = { ...data.job, items: data.items }; + setActiveJob(jobWithItems); + + // Also update the job in the list + setJobs((prev) => + prev.map((j) => (j.id === jobId ? data.job : j)) + ); + + return jobWithItems; + }, []); + + /** Import a playlist: creates a job + download_items rows. */ + const importPlaylist = useCallback( + async (playlistUrl: string, genre?: MusicGenre) => { + setIsImporting(true); + try { + const { data, error } = await supabase.functions.invoke( + "youtube-playlist", + { body: { playlistUrl, genre: genre || null } } + ); + if (error) throw new Error(error.message ?? "Import failed"); + if (data?.error) throw new Error(data.error); + + // Refresh the jobs list and select the new job + await fetchJobs(); + if (data.jobId) { + await refreshStatus(data.jobId); + } + + return data as { + jobId: string; + playlistTitle: string; + totalItems: number; + }; + } finally { + setIsImporting(false); + } + }, + [fetchJobs, refreshStatus] + ); + + /** Process all pending items for a job, one at a time. */ + const startProcessing = useCallback( + async (jobId: string) => { + // Abort any existing processing loop + if (abortRef.current) { + abortRef.current.abort(); + } + + const controller = new AbortController(); + abortRef.current = controller; + setIsProcessing(true); + + try { + let done = false; + + while (!done && !controller.signal.aborted) { + const { data, error } = await supabase.functions.invoke( + "youtube-process", + { body: { jobId } } + ); + + if (error) throw new Error(error.message ?? "Processing failed"); + if (data?.error) throw new Error(data.error); + + done = data.done === true; + + // Refresh the job status to get updated items + await refreshStatus(jobId); + + // Delay between calls to avoid hammering the function + if (!done && !controller.signal.aborted) { + await new Promise((resolve, reject) => { + const timer = setTimeout(resolve, PROCESS_DELAY_MS); + controller.signal.addEventListener( + "abort", + () => { + clearTimeout(timer); + reject(new DOMException("Aborted", "AbortError")); + }, + { once: true } + ); + }); + } + } + } catch (err) { + if (err instanceof DOMException && err.name === "AbortError") { + // Graceful stop — not an error + return; + } + throw err; + } finally { + setIsProcessing(false); + abortRef.current = null; + // Final refresh to get latest state + await refreshStatus(jobId).catch(() => {}); + } + }, + [refreshStatus] + ); + + /** Stop the current processing loop. */ + const stopProcessing = useCallback(() => { + if (abortRef.current) { + abortRef.current.abort(); + abortRef.current = null; + } + }, []); + + /** Delete a job, its items, and associated storage files. */ + const deleteJob = useCallback( + async (jobId: string) => { + await invokeDelete<{ deleted: boolean }>("youtube-status", { jobId }); + + // Remove the job from local state + setJobs((prev) => prev.filter((j) => j.id !== jobId)); + + // Clear active job if it was the deleted one + setActiveJob((prev) => (prev?.id === jobId ? null : prev)); + }, + [] + ); + + /** Update the genre on a single download item. */ + const updateItemGenre = useCallback( + async (itemId: string, genre: MusicGenre | null) => { + await invokePatch<{ updated: boolean }>("youtube-status", { + itemId, + genre, + }); + + // Update local state + setActiveJob((prev) => { + if (!prev) return prev; + return { + ...prev, + items: prev.items.map((item) => + item.id === itemId ? { ...item, genre } : item + ), + }; + }); + }, + [] + ); + + return { + jobs, + activeJob, + isProcessing, + isImporting, + fetchJobs, + refreshStatus, + importPlaylist, + startProcessing, + stopProcessing, + deleteJob, + updateItemGenre, + }; +} diff --git a/scripts/deploy-functions.sh b/scripts/deploy-functions.sh new file mode 100755 index 0000000..e336bc7 --- /dev/null +++ b/scripts/deploy-functions.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +set -euo pipefail + +# ── Configuration ────────────────────────────────────────────── +DEPLOY_HOST="${DEPLOY_HOST:-1000co.fr}" +DEPLOY_USER="${DEPLOY_USER:-millian}" +DEPLOY_PATH="${DEPLOY_PATH:-/opt/supabase/volumes/functions}" +WORKER_PATH="${WORKER_PATH:-/opt/supabase/youtube-worker}" +# ─────────────────────────────────────────────────────────────── + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +FUNCTIONS_DIR="$SCRIPT_DIR/../supabase/functions" +WORKER_DIR="$SCRIPT_DIR/../youtube-worker" + +# ── Deploy edge functions ────────────────────────────────────── +echo "==> Deploying edge functions" +rsync -avz --delete \ + --exclude='node_modules' \ + --exclude='.DS_Store' \ + "$FUNCTIONS_DIR/" \ + "$DEPLOY_USER@$DEPLOY_HOST:$DEPLOY_PATH/" + +echo "Restarting supabase-edge-functions..." +ssh "$DEPLOY_USER@$DEPLOY_HOST" "docker restart supabase-edge-functions" + +# ── Deploy youtube-worker sidecar ────────────────────────────── +echo "" +echo "==> Deploying youtube-worker sidecar" +rsync -avz --delete \ + --exclude='node_modules' \ + --exclude='.DS_Store' \ + "$WORKER_DIR/" \ + "$DEPLOY_USER@$DEPLOY_HOST:$WORKER_PATH/" + +echo "Building and restarting youtube-worker..." +ssh "$DEPLOY_USER@$DEPLOY_HOST" "\ + cd $WORKER_PATH && \ + docker build -t youtube-worker:latest . && \ + docker stop youtube-worker 2>/dev/null || true && \ + docker rm youtube-worker 2>/dev/null || true && \ + docker run -d \ + --name youtube-worker \ + --restart unless-stopped \ + --network supabase_supabase-network \ + -e SUPABASE_URL=\$(docker exec supabase-edge-functions printenv SUPABASE_URL) \ + -e SUPABASE_SERVICE_ROLE_KEY=\$(docker exec supabase-edge-functions printenv SUPABASE_SERVICE_ROLE_KEY) \ + -e SUPABASE_PUBLIC_URL=https://supabase.1000co.fr \ + -e STORAGE_BUCKET=workout-audio \ + -e PORT=3001 \ + youtube-worker:latest" + +echo "" +echo "Done. Verifying youtube-worker health..." +sleep 3 +ssh "$DEPLOY_USER@$DEPLOY_HOST" "docker logs youtube-worker --tail 5" diff --git a/supabase/functions/_shared/auth.ts b/supabase/functions/_shared/auth.ts new file mode 100644 index 0000000..dd49b14 --- /dev/null +++ b/supabase/functions/_shared/auth.ts @@ -0,0 +1,62 @@ +import { createClient } from "@supabase/supabase-js"; + +/** + * Verifies the request comes from an authenticated admin user. + * Extracts the JWT from the Authorization header, validates it, + * and checks the user exists in the admin_users table. + * + * Returns the authenticated user's ID. + * Throws an error with status code if auth fails. + */ +export async function verifyAdmin(req: Request): Promise { + const authHeader = req.headers.get("Authorization"); + if (!authHeader?.startsWith("Bearer ")) { + throw new AuthError("Missing or invalid Authorization header", 401); + } + + const token = authHeader.replace("Bearer ", ""); + + const url = Deno.env.get("SUPABASE_URL"); + const anonKey = Deno.env.get("SUPABASE_ANON_KEY"); + + if (!url || !anonKey) { + throw new AuthError("Missing SUPABASE_URL or SUPABASE_ANON_KEY", 500); + } + + // Create a user-scoped client with the provided JWT + const supabase = createClient(url, anonKey, { + global: { headers: { Authorization: `Bearer ${token}` } }, + auth: { persistSession: false }, + }); + + const { + data: { user }, + error: userError, + } = await supabase.auth.getUser(); + + if (userError || !user) { + throw new AuthError("Invalid or expired token", 401); + } + + // Verify user is an admin + const { data: adminUser, error: adminError } = await supabase + .from("admin_users") + .select("id") + .eq("id", user.id) + .single(); + + if (adminError || !adminUser) { + throw new AuthError("User is not an admin", 403); + } + + return user.id; +} + +export class AuthError extends Error { + status: number; + constructor(message: string, status: number) { + super(message); + this.name = "AuthError"; + this.status = status; + } +} diff --git a/supabase/functions/_shared/cors.ts b/supabase/functions/_shared/cors.ts new file mode 100644 index 0000000..b9de29e --- /dev/null +++ b/supabase/functions/_shared/cors.ts @@ -0,0 +1,13 @@ +export const corsHeaders = { + "Access-Control-Allow-Origin": "*", + "Access-Control-Allow-Headers": + "authorization, x-client-info, apikey, content-type", + "Access-Control-Allow-Methods": "POST, GET, OPTIONS", +}; + +export function handleCors(req: Request): Response | null { + if (req.method === "OPTIONS") { + return new Response("ok", { headers: corsHeaders }); + } + return null; +} diff --git a/supabase/functions/_shared/supabase-client.ts b/supabase/functions/_shared/supabase-client.ts new file mode 100644 index 0000000..228e5d7 --- /dev/null +++ b/supabase/functions/_shared/supabase-client.ts @@ -0,0 +1,19 @@ +import { createClient, SupabaseClient } from "@supabase/supabase-js"; + +/** + * Creates a Supabase client using the service role key. + * This bypasses RLS and is used for server-side operations in edge functions. + * Auth is verified separately via the admin auth helper. + */ +export function createServiceClient(): SupabaseClient { + const url = Deno.env.get("SUPABASE_URL"); + const key = Deno.env.get("SUPABASE_SERVICE_ROLE_KEY"); + + if (!url || !key) { + throw new Error("Missing SUPABASE_URL or SUPABASE_SERVICE_ROLE_KEY"); + } + + return createClient(url, key, { + auth: { persistSession: false }, + }); +} diff --git a/supabase/functions/_shared/youtube-client.ts b/supabase/functions/_shared/youtube-client.ts new file mode 100644 index 0000000..db5011f --- /dev/null +++ b/supabase/functions/_shared/youtube-client.ts @@ -0,0 +1,88 @@ +/** + * YouTube client that delegates to the youtube-worker sidecar container. + * The sidecar runs youtubei.js in a full Node.js environment without + * CPU time limits. + */ + +const WORKER_URL = + Deno.env.get("YOUTUBE_WORKER_URL") || "http://youtube-worker:3001"; + +export interface PlaylistItem { + videoId: string; + title: string; + durationSeconds: number; + thumbnailUrl: string | null; +} + +export interface PlaylistInfo { + title: string; + items: PlaylistItem[]; +} + +export interface DownloadResult { + storagePath: string; + publicUrl: string; +} + +async function workerFetch( + path: string, + body: Record +): Promise { + const res = await fetch(`${WORKER_URL}${path}`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(body), + }); + + const data = await res.json(); + + if (!res.ok) { + throw new Error(data.error || `Worker responded with ${res.status}`); + } + + return data as T; +} + +/** + * Parses a YouTube playlist ID from various URL formats. + */ +export function parsePlaylistId(url: string): string { + if (/^[A-Za-z0-9_-]+$/.test(url) && !url.includes(".")) { + return url; + } + + try { + const parsed = new URL(url); + const listParam = parsed.searchParams.get("list"); + if (listParam) { + return listParam; + } + } catch { + // Not a valid URL + } + + throw new Error( + `Could not parse playlist ID from: "${url}". ` + + `Expected a YouTube playlist URL or raw playlist ID.` + ); +} + +/** + * Fetches playlist metadata and items via the sidecar. + */ +export async function getPlaylistItems( + playlistUrl: string +): Promise { + return workerFetch("/playlist", { playlistUrl }); +} + +/** + * Downloads audio and uploads to Supabase Storage via the sidecar. + * Returns the storage path and public URL. + */ +export async function downloadAndUploadAudio( + videoId: string, + jobId: string +): Promise { + return workerFetch("/download", { videoId, jobId }); +} diff --git a/supabase/functions/deno.json b/supabase/functions/deno.json new file mode 100644 index 0000000..ec275fc --- /dev/null +++ b/supabase/functions/deno.json @@ -0,0 +1,8 @@ +{ + "imports": { + "@supabase/supabase-js": "https://esm.sh/@supabase/supabase-js@2" + }, + "compilerOptions": { + "strict": true + } +} diff --git a/supabase/functions/main/index.ts b/supabase/functions/main/index.ts new file mode 100644 index 0000000..d873432 --- /dev/null +++ b/supabase/functions/main/index.ts @@ -0,0 +1,95 @@ +import * as jose from "https://deno.land/x/jose@v4.14.4/index.ts"; + +console.log("main function started"); + +const JWT_SECRET = Deno.env.get("JWT_SECRET"); +const VERIFY_JWT = Deno.env.get("VERIFY_JWT") === "true"; + +function getAuthToken(req: Request) { + const authHeader = req.headers.get("authorization"); + if (!authHeader) { + throw new Error("Missing authorization header"); + } + const [bearer, token] = authHeader.split(" "); + if (bearer !== "Bearer") { + throw new Error(`Auth header is not 'Bearer {token}'`); + } + return token; +} + +async function isValidJWT(jwt: string): Promise { + if (!JWT_SECRET) { + console.error("JWT_SECRET not available for token verification"); + return false; + } + const encoder = new TextEncoder(); + const secretKey = encoder.encode(JWT_SECRET); + try { + await jose.jwtVerify(jwt, secretKey); + } catch (e) { + console.error("JWT verification error", e); + return false; + } + return true; +} + +Deno.serve(async (req: Request) => { + if (req.method !== "OPTIONS" && VERIFY_JWT) { + try { + const token = getAuthToken(req); + const valid = await isValidJWT(token); + if (!valid) { + return new Response(JSON.stringify({ msg: "Invalid JWT" }), { + status: 401, + headers: { "Content-Type": "application/json" }, + }); + } + } catch (e) { + console.error(e); + return new Response(JSON.stringify({ msg: e.toString() }), { + status: 401, + headers: { "Content-Type": "application/json" }, + }); + } + } + + const url = new URL(req.url); + const { pathname } = url; + const path_parts = pathname.split("/"); + const service_name = path_parts[1]; + + if (!service_name || service_name === "") { + return new Response( + JSON.stringify({ msg: "missing function name in request" }), + { status: 400, headers: { "Content-Type": "application/json" } } + ); + } + + const servicePath = `/home/deno/functions/${service_name}`; + console.error(`serving the request with ${servicePath}`); + + const memoryLimitMb = 512; + const workerTimeoutMs = 10 * 60 * 1000; + const noModuleCache = false; + const importMapPath = null; + const envVarsObj = Deno.env.toObject(); + const envVars = Object.keys(envVarsObj).map((k) => [k, envVarsObj[k]]); + + try { + const worker = await EdgeRuntime.userWorkers.create({ + servicePath, + memoryLimitMb, + workerTimeoutMs, + noModuleCache, + importMapPath, + envVars, + }); + return await worker.fetch(req); + } catch (e) { + const error = { msg: e.toString() }; + return new Response(JSON.stringify(error), { + status: 500, + headers: { "Content-Type": "application/json" }, + }); + } +}); diff --git a/supabase/functions/youtube-playlist/index.ts b/supabase/functions/youtube-playlist/index.ts new file mode 100644 index 0000000..47b575a --- /dev/null +++ b/supabase/functions/youtube-playlist/index.ts @@ -0,0 +1,124 @@ +import { corsHeaders, handleCors } from "../_shared/cors.ts"; +import { verifyAdmin, AuthError } from "../_shared/auth.ts"; +import { createServiceClient } from "../_shared/supabase-client.ts"; +import { + parsePlaylistId, + getPlaylistItems, +} from "../_shared/youtube-client.ts"; + +Deno.serve(async (req: Request) => { + // Handle CORS preflight + const corsResponse = handleCors(req); + if (corsResponse) return corsResponse; + + try { + // Only accept POST + if (req.method !== "POST") { + return new Response(JSON.stringify({ error: "Method not allowed" }), { + status: 405, + headers: { ...corsHeaders, "Content-Type": "application/json" }, + }); + } + + // Verify admin auth + const userId = await verifyAdmin(req); + + // Parse request body + const { playlistUrl, genre } = await req.json(); + if (!playlistUrl || typeof playlistUrl !== "string") { + return new Response( + JSON.stringify({ error: "playlistUrl is required" }), + { + status: 400, + headers: { ...corsHeaders, "Content-Type": "application/json" }, + } + ); + } + + // Validate the URL format + parsePlaylistId(playlistUrl); + + // Fetch playlist metadata via the sidecar worker + const playlist = await getPlaylistItems(playlistUrl); + + if (playlist.items.length === 0) { + return new Response( + JSON.stringify({ error: "Playlist is empty or could not be loaded" }), + { + status: 400, + headers: { ...corsHeaders, "Content-Type": "application/json" }, + } + ); + } + + // Create the download job in the database + const supabase = createServiceClient(); + + const { data: job, error: jobError } = await supabase + .from("download_jobs") + .insert({ + playlist_url: playlistUrl, + playlist_title: playlist.title, + status: "pending", + total_items: playlist.items.length, + completed_items: 0, + failed_items: 0, + created_by: userId, + }) + .select() + .single(); + + if (jobError || !job) { + throw new Error(`Failed to create download job: ${jobError?.message}`); + } + + // Create download items for each video + const itemRows = playlist.items.map((item) => ({ + job_id: job.id, + video_id: item.videoId, + title: item.title, + duration_seconds: item.durationSeconds, + thumbnail_url: item.thumbnailUrl, + status: "pending", + genre: genre || null, + })); + + const { error: itemsError } = await supabase + .from("download_items") + .insert(itemRows); + + if (itemsError) { + await supabase.from("download_jobs").delete().eq("id", job.id); + throw new Error(`Failed to create download items: ${itemsError.message}`); + } + + // Fetch the created items to return + const { data: items } = await supabase + .from("download_items") + .select("*") + .eq("job_id", job.id) + .order("created_at", { ascending: true }); + + return new Response( + JSON.stringify({ + jobId: job.id, + playlistTitle: playlist.title, + totalItems: playlist.items.length, + items: items ?? [], + }), + { + status: 200, + headers: { ...corsHeaders, "Content-Type": "application/json" }, + } + ); + } catch (error) { + const status = error instanceof AuthError ? error.status : 500; + const message = + error instanceof Error ? error.message : "Unknown error occurred"; + + return new Response(JSON.stringify({ error: message }), { + status, + headers: { ...corsHeaders, "Content-Type": "application/json" }, + }); + } +}); diff --git a/supabase/functions/youtube-process/index.ts b/supabase/functions/youtube-process/index.ts new file mode 100644 index 0000000..c386c0e --- /dev/null +++ b/supabase/functions/youtube-process/index.ts @@ -0,0 +1,200 @@ +import { corsHeaders, handleCors } from "../_shared/cors.ts"; +import { verifyAdmin, AuthError } from "../_shared/auth.ts"; +import { createServiceClient } from "../_shared/supabase-client.ts"; +import { downloadAndUploadAudio } from "../_shared/youtube-client.ts"; + +Deno.serve(async (req: Request) => { + // Handle CORS preflight + const corsResponse = handleCors(req); + if (corsResponse) return corsResponse; + + try { + // Only accept POST + if (req.method !== "POST") { + return new Response(JSON.stringify({ error: "Method not allowed" }), { + status: 405, + headers: { ...corsHeaders, "Content-Type": "application/json" }, + }); + } + + // Verify admin auth + await verifyAdmin(req); + + // Parse request body + const { jobId } = await req.json(); + if (!jobId || typeof jobId !== "string") { + return new Response(JSON.stringify({ error: "jobId is required" }), { + status: 400, + headers: { ...corsHeaders, "Content-Type": "application/json" }, + }); + } + + const supabase = createServiceClient(); + + // Get the job + const { data: job, error: jobError } = await supabase + .from("download_jobs") + .select("*") + .eq("id", jobId) + .single(); + + if (jobError || !job) { + return new Response(JSON.stringify({ error: "Job not found" }), { + status: 404, + headers: { ...corsHeaders, "Content-Type": "application/json" }, + }); + } + + // Get the next pending item + const { data: item, error: itemError } = await supabase + .from("download_items") + .select("*") + .eq("job_id", jobId) + .eq("status", "pending") + .order("created_at", { ascending: true }) + .limit(1) + .single(); + + // No pending items -- finalize the job + if (itemError || !item) { + const finalStatus = + job.completed_items > 0 ? "completed" : "failed"; + + await supabase + .from("download_jobs") + .update({ status: finalStatus }) + .eq("id", jobId); + + return new Response( + JSON.stringify({ + status: finalStatus, + progress: { + completed: job.completed_items, + failed: job.failed_items, + total: job.total_items, + }, + processedItem: null, + done: true, + }), + { + status: 200, + headers: { ...corsHeaders, "Content-Type": "application/json" }, + } + ); + } + + // Mark item as downloading, job as processing + await Promise.all([ + supabase + .from("download_items") + .update({ status: "downloading" }) + .eq("id", item.id), + supabase + .from("download_jobs") + .update({ status: "processing" }) + .eq("id", jobId), + ]); + + try { + // Download audio + upload to storage via the sidecar worker + const { storagePath, publicUrl } = await downloadAndUploadAudio( + item.video_id, + jobId + ); + + // Mark item as completed + await supabase + .from("download_items") + .update({ + status: "completed", + storage_path: storagePath, + public_url: publicUrl, + }) + .eq("id", item.id); + + // Increment completed count on job + await supabase + .from("download_jobs") + .update({ + completed_items: job.completed_items + 1, + }) + .eq("id", jobId); + + return new Response( + JSON.stringify({ + status: "processing", + progress: { + completed: job.completed_items + 1, + failed: job.failed_items, + total: job.total_items, + }, + processedItem: { + id: item.id, + videoId: item.video_id, + title: item.title, + status: "completed", + publicUrl, + storagePath, + }, + done: false, + }), + { + status: 200, + headers: { ...corsHeaders, "Content-Type": "application/json" }, + } + ); + } catch (downloadError) { + const errorMessage = + downloadError instanceof Error + ? downloadError.message + : "Unknown download error"; + + await supabase + .from("download_items") + .update({ + status: "failed", + error_message: errorMessage, + }) + .eq("id", item.id); + + await supabase + .from("download_jobs") + .update({ + failed_items: job.failed_items + 1, + }) + .eq("id", jobId); + + return new Response( + JSON.stringify({ + status: "processing", + progress: { + completed: job.completed_items, + failed: job.failed_items + 1, + total: job.total_items, + }, + processedItem: { + id: item.id, + videoId: item.video_id, + title: item.title, + status: "failed", + error: errorMessage, + }, + done: false, + }), + { + status: 200, + headers: { ...corsHeaders, "Content-Type": "application/json" }, + } + ); + } + } catch (error) { + const status = error instanceof AuthError ? error.status : 500; + const message = + error instanceof Error ? error.message : "Unknown error occurred"; + + return new Response(JSON.stringify({ error: message }), { + status, + headers: { ...corsHeaders, "Content-Type": "application/json" }, + }); + } +}); diff --git a/supabase/functions/youtube-status/index.ts b/supabase/functions/youtube-status/index.ts new file mode 100644 index 0000000..9e50163 --- /dev/null +++ b/supabase/functions/youtube-status/index.ts @@ -0,0 +1,171 @@ +import { corsHeaders, handleCors } from "../_shared/cors.ts"; +import { verifyAdmin, AuthError } from "../_shared/auth.ts"; +import { createServiceClient } from "../_shared/supabase-client.ts"; + +Deno.serve(async (req: Request) => { + // Handle CORS preflight + const corsResponse = handleCors(req); + if (corsResponse) return corsResponse; + + try { + // Verify admin auth + await verifyAdmin(req); + + const supabase = createServiceClient(); + + // ── DELETE: remove a job, its items, and storage files ───── + if (req.method === "DELETE") { + const { jobId } = await req.json(); + if (!jobId || typeof jobId !== "string") { + return new Response(JSON.stringify({ error: "jobId is required" }), { + status: 400, + headers: { ...corsHeaders, "Content-Type": "application/json" }, + }); + } + + // Fetch items to find storage paths for cleanup + const { data: items } = await supabase + .from("download_items") + .select("storage_path") + .eq("job_id", jobId); + + // Delete files from storage bucket + const storagePaths = (items ?? []) + .map((i) => i.storage_path) + .filter(Boolean) as string[]; + + if (storagePaths.length > 0) { + const { error: storageError } = await supabase.storage + .from("workout-audio") + .remove(storagePaths); + + if (storageError) { + console.error("Storage cleanup error:", storageError.message); + // Continue with DB deletion even if storage cleanup fails + } + } + + // Delete the job row (items cascade via FK) + const { error: deleteError } = await supabase + .from("download_jobs") + .delete() + .eq("id", jobId); + + if (deleteError) { + throw new Error(`Failed to delete job: ${deleteError.message}`); + } + + return new Response( + JSON.stringify({ deleted: true, jobId, filesRemoved: storagePaths.length }), + { + status: 200, + headers: { ...corsHeaders, "Content-Type": "application/json" }, + } + ); + } + + // ── GET: fetch job(s) ───────────────────────────────────── + if (req.method !== "GET" && req.method !== "PATCH") { + return new Response(JSON.stringify({ error: "Method not allowed" }), { + status: 405, + headers: { ...corsHeaders, "Content-Type": "application/json" }, + }); + } + + // ── PATCH: update genre on download items ───────────────── + if (req.method === "PATCH") { + const { itemId, genre } = await req.json(); + if (!itemId || typeof itemId !== "string") { + return new Response( + JSON.stringify({ error: "itemId is required" }), + { + status: 400, + headers: { ...corsHeaders, "Content-Type": "application/json" }, + } + ); + } + + // genre can be null (to clear) or a valid genre string + const { error: updateError } = await supabase + .from("download_items") + .update({ genre: genre || null }) + .eq("id", itemId); + + if (updateError) { + throw new Error(`Failed to update genre: ${updateError.message}`); + } + + return new Response( + JSON.stringify({ updated: true, itemId, genre: genre || null }), + { + status: 200, + headers: { ...corsHeaders, "Content-Type": "application/json" }, + } + ); + } + + // ── GET: fetch job(s) ───────────────────────────────────── + + // Parse query params + const url = new URL(req.url); + const jobId = url.searchParams.get("jobId"); + + // If jobId is provided, return that specific job + items + if (jobId) { + const { data: job, error: jobError } = await supabase + .from("download_jobs") + .select("*") + .eq("id", jobId) + .single(); + + if (jobError || !job) { + return new Response(JSON.stringify({ error: "Job not found" }), { + status: 404, + headers: { ...corsHeaders, "Content-Type": "application/json" }, + }); + } + + const { data: items } = await supabase + .from("download_items") + .select("*") + .eq("job_id", jobId) + .order("created_at", { ascending: true }); + + return new Response( + JSON.stringify({ job, items: items ?? [] }), + { + status: 200, + headers: { ...corsHeaders, "Content-Type": "application/json" }, + } + ); + } + + // No jobId -- return all jobs (most recent first) + const { data: jobs, error: jobsError } = await supabase + .from("download_jobs") + .select("*") + .order("created_at", { ascending: false }) + .limit(50); + + if (jobsError) { + throw new Error(`Failed to fetch jobs: ${jobsError.message}`); + } + + return new Response( + JSON.stringify({ jobs: jobs ?? [] }), + { + status: 200, + headers: { ...corsHeaders, "Content-Type": "application/json" }, + } + ); + } catch (error) { + const status = error instanceof AuthError ? error.status : 500; + const message = + error instanceof Error ? error.message : "Unknown error occurred"; + + return new Response(JSON.stringify({ error: message }), { + status, + headers: { ...corsHeaders, "Content-Type": "application/json" }, + }); + } +}); diff --git a/supabase/migrations/002_download_jobs.sql b/supabase/migrations/002_download_jobs.sql new file mode 100644 index 0000000..238a34d --- /dev/null +++ b/supabase/migrations/002_download_jobs.sql @@ -0,0 +1,96 @@ +-- ============================================================ +-- YouTube Playlist Download Jobs +-- ============================================================ +-- Tracks playlist-level download jobs and per-video items +-- Used by the youtube-playlist, youtube-process, youtube-status edge functions + +-- Download jobs table (one row per playlist import) +CREATE TABLE IF NOT EXISTS public.download_jobs ( + id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + playlist_url TEXT NOT NULL, + playlist_title TEXT, + status TEXT NOT NULL DEFAULT 'pending' + CHECK (status IN ('pending', 'processing', 'completed', 'failed')), + total_items INTEGER NOT NULL DEFAULT 0, + completed_items INTEGER NOT NULL DEFAULT 0, + failed_items INTEGER NOT NULL DEFAULT 0, + created_by UUID NOT NULL REFERENCES auth.users(id) ON DELETE CASCADE, + created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(), + updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW() +); + +-- Download items table (one row per video in the playlist) +CREATE TABLE IF NOT EXISTS public.download_items ( + id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + job_id UUID NOT NULL REFERENCES public.download_jobs(id) ON DELETE CASCADE, + video_id TEXT NOT NULL, + title TEXT, + duration_seconds INTEGER, + thumbnail_url TEXT, + status TEXT NOT NULL DEFAULT 'pending' + CHECK (status IN ('pending', 'downloading', 'completed', 'failed')), + storage_path TEXT, + public_url TEXT, + error_message TEXT, + created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW() +); + +-- ============================================================ +-- INDEXES +-- ============================================================ + +CREATE INDEX IF NOT EXISTS idx_download_jobs_created_by ON public.download_jobs(created_by); +CREATE INDEX IF NOT EXISTS idx_download_jobs_status ON public.download_jobs(status); +CREATE INDEX IF NOT EXISTS idx_download_jobs_created_at ON public.download_jobs(created_at DESC); +CREATE INDEX IF NOT EXISTS idx_download_items_job_id ON public.download_items(job_id); +CREATE INDEX IF NOT EXISTS idx_download_items_status ON public.download_items(status); +CREATE INDEX IF NOT EXISTS idx_download_items_created_at ON public.download_items(created_at); + +-- ============================================================ +-- TRIGGERS +-- ============================================================ + +CREATE TRIGGER update_download_jobs_updated_at + BEFORE UPDATE ON public.download_jobs + FOR EACH ROW EXECUTE FUNCTION public.update_updated_at_column(); + +-- ============================================================ +-- ROW LEVEL SECURITY +-- ============================================================ + +ALTER TABLE public.download_jobs ENABLE ROW LEVEL SECURITY; +ALTER TABLE public.download_items ENABLE ROW LEVEL SECURITY; + +-- Admin-only read/write (same pattern as existing tables) +CREATE POLICY "Allow admin read access" ON public.download_jobs + FOR SELECT USING ( + EXISTS (SELECT 1 FROM public.admin_users WHERE id = auth.uid()) + ); + +CREATE POLICY "Allow admin write access" ON public.download_jobs + FOR ALL USING ( + EXISTS (SELECT 1 FROM public.admin_users WHERE id = auth.uid()) + ); + +CREATE POLICY "Allow admin read access" ON public.download_items + FOR SELECT USING ( + EXISTS (SELECT 1 FROM public.admin_users WHERE id = auth.uid()) + ); + +CREATE POLICY "Allow admin write access" ON public.download_items + FOR ALL USING ( + EXISTS (SELECT 1 FROM public.admin_users WHERE id = auth.uid()) + ); + +-- ============================================================ +-- STORAGE BUCKET: workout-audio +-- ============================================================ +-- Create via Supabase Dashboard or CLI: +-- supabase storage create workout-audio --public +-- +-- Configuration: +-- Bucket name: workout-audio +-- Public: true +-- Allowed MIME types: audio/mp4, audio/webm, audio/mpeg +-- Max file size: 100MB +-- Path pattern: {job_id}/{video_id}.m4a diff --git a/supabase/migrations/003_music_genre.sql b/supabase/migrations/003_music_genre.sql new file mode 100644 index 0000000..35b3ccf --- /dev/null +++ b/supabase/migrations/003_music_genre.sql @@ -0,0 +1,18 @@ +-- ============================================================ +-- Music Genre Classification +-- ============================================================ +-- Adds per-track genre to download_items so users can filter +-- workout music by genre preference. + +-- Genre enum values: edm, hip-hop, pop, rock, latin, house, +-- drum-and-bass, dubstep, r-and-b, country, metal, ambient + +ALTER TABLE public.download_items + ADD COLUMN IF NOT EXISTS genre TEXT + CHECK (genre IS NULL OR genre IN ( + 'edm', 'hip-hop', 'pop', 'rock', 'latin', 'house', + 'drum-and-bass', 'dubstep', 'r-and-b', 'country', 'metal', 'ambient' + )); + +CREATE INDEX IF NOT EXISTS idx_download_items_genre + ON public.download_items(genre); diff --git a/youtube-worker/Dockerfile b/youtube-worker/Dockerfile new file mode 100644 index 0000000..d975ddf --- /dev/null +++ b/youtube-worker/Dockerfile @@ -0,0 +1,21 @@ +FROM node:20-slim + +# Install yt-dlp (standalone binary) and ffmpeg for audio extraction +RUN apt-get update && \ + apt-get install -y --no-install-recommends python3 ffmpeg curl ca-certificates && \ + curl -L https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp -o /usr/local/bin/yt-dlp && \ + chmod +x /usr/local/bin/yt-dlp && \ + apt-get remove -y curl && \ + apt-get autoremove -y && \ + rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +COPY package.json package-lock.json* ./ +RUN npm install --production + +COPY server.js ./ + +EXPOSE 3001 + +CMD ["node", "server.js"] diff --git a/youtube-worker/package-lock.json b/youtube-worker/package-lock.json new file mode 100644 index 0000000..93c5860 --- /dev/null +++ b/youtube-worker/package-lock.json @@ -0,0 +1,190 @@ +{ + "name": "youtube-worker", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "youtube-worker", + "version": "1.0.0", + "dependencies": { + "@supabase/supabase-js": "^2.49.1", + "youtubei.js": "^17.0.1" + } + }, + "node_modules/@bufbuild/protobuf": { + "version": "2.11.0", + "resolved": "https://registry.npmjs.org/@bufbuild/protobuf/-/protobuf-2.11.0.tgz", + "integrity": "sha512-sBXGT13cpmPR5BMgHE6UEEfEaShh5Ror6rfN3yEK5si7QVrtZg8LEPQb0VVhiLRUslD2yLnXtnRzG035J/mZXQ==", + "license": "(Apache-2.0 AND BSD-3-Clause)" + }, + "node_modules/@supabase/auth-js": { + "version": "2.100.0", + "resolved": "https://registry.npmjs.org/@supabase/auth-js/-/auth-js-2.100.0.tgz", + "integrity": "sha512-pdT3ye3UVRN1Cg0wom6BmyY+XTtp5DiJaYnPi6j8ht5i8Lq8kfqxJMJz9GI9YDKk3w1nhGOPnh6Qz5qpyYm+1w==", + "license": "MIT", + "dependencies": { + "tslib": "2.8.1" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@supabase/functions-js": { + "version": "2.100.0", + "resolved": "https://registry.npmjs.org/@supabase/functions-js/-/functions-js-2.100.0.tgz", + "integrity": "sha512-keLg79RPwP+uiwHuxFPTFgDRxPV46LM4j/swjyR2GKJgWniTVSsgiBHfbIBDcrQwehLepy09b/9QSHUywtKRWQ==", + "license": "MIT", + "dependencies": { + "tslib": "2.8.1" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@supabase/phoenix": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/@supabase/phoenix/-/phoenix-0.4.0.tgz", + "integrity": "sha512-RHSx8bHS02xwfHdAbX5Lpbo6PXbgyf7lTaXTlwtFDPwOIw64NnVRwFAXGojHhjtVYI+PEPNSWwkL90f4agN3bw==", + "license": "MIT" + }, + "node_modules/@supabase/postgrest-js": { + "version": "2.100.0", + "resolved": "https://registry.npmjs.org/@supabase/postgrest-js/-/postgrest-js-2.100.0.tgz", + "integrity": "sha512-xYNvNbBJaXOGcrZ44wxwp5830uo1okMHGS8h8dm3u4f0xcZ39yzbryUsubTJW41MG2gbL/6U57cA4Pi6YMZ9pA==", + "license": "MIT", + "dependencies": { + "tslib": "2.8.1" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@supabase/realtime-js": { + "version": "2.100.0", + "resolved": "https://registry.npmjs.org/@supabase/realtime-js/-/realtime-js-2.100.0.tgz", + "integrity": "sha512-2AZs00zzEF0HuCKY8grz5eCYlwEfVi5HONLZFoNR6aDfxQivl8zdQYNjyFoqN2MZiVhQHD7u6XV/xHwM8mCEHw==", + "license": "MIT", + "dependencies": { + "@supabase/phoenix": "^0.4.0", + "@types/ws": "^8.18.1", + "tslib": "2.8.1", + "ws": "^8.18.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@supabase/storage-js": { + "version": "2.100.0", + "resolved": "https://registry.npmjs.org/@supabase/storage-js/-/storage-js-2.100.0.tgz", + "integrity": "sha512-d4EeuK6RNIgYNA2MU9kj8lQrLm5AzZ+WwpWjGkii6SADQNIGTC/uiaTRu02XJ5AmFALQfo8fLl9xuCkO6Xw+iQ==", + "license": "MIT", + "dependencies": { + "iceberg-js": "^0.8.1", + "tslib": "2.8.1" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@supabase/supabase-js": { + "version": "2.100.0", + "resolved": "https://registry.npmjs.org/@supabase/supabase-js/-/supabase-js-2.100.0.tgz", + "integrity": "sha512-r0tlcukejJXJ1m/2eG/Ya5eYs4W8AC7oZfShpG3+SIo/eIU9uIt76ZeYI1SoUwUmcmzlAbgch+HDZDR/toVQPQ==", + "license": "MIT", + "dependencies": { + "@supabase/auth-js": "2.100.0", + "@supabase/functions-js": "2.100.0", + "@supabase/postgrest-js": "2.100.0", + "@supabase/realtime-js": "2.100.0", + "@supabase/storage-js": "2.100.0" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@types/node": { + "version": "25.5.0", + "resolved": "https://registry.npmjs.org/@types/node/-/node-25.5.0.tgz", + "integrity": "sha512-jp2P3tQMSxWugkCUKLRPVUpGaL5MVFwF8RDuSRztfwgN1wmqJeMSbKlnEtQqU8UrhTmzEmZdu2I6v2dpp7XIxw==", + "license": "MIT", + "dependencies": { + "undici-types": "~7.18.0" + } + }, + "node_modules/@types/ws": { + "version": "8.18.1", + "resolved": "https://registry.npmjs.org/@types/ws/-/ws-8.18.1.tgz", + "integrity": "sha512-ThVF6DCVhA8kUGy+aazFQ4kXQ7E1Ty7A3ypFOe0IcJV8O/M511G99AW24irKrW56Wt44yG9+ij8FaqoBGkuBXg==", + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/iceberg-js": { + "version": "0.8.1", + "resolved": "https://registry.npmjs.org/iceberg-js/-/iceberg-js-0.8.1.tgz", + "integrity": "sha512-1dhVQZXhcHje7798IVM+xoo/1ZdVfzOMIc8/rgVSijRK38EDqOJoGula9N/8ZI5RD8QTxNQtK/Gozpr+qUqRRA==", + "license": "MIT", + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/meriyah": { + "version": "6.1.4", + "resolved": "https://registry.npmjs.org/meriyah/-/meriyah-6.1.4.tgz", + "integrity": "sha512-Sz8FzjzI0kN13GK/6MVEsVzMZEPvOhnmmI1lU5+/1cGOiK3QUahntrNNtdVeihrO7t9JpoH75iMNXg6R6uWflQ==", + "license": "ISC", + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "license": "0BSD" + }, + "node_modules/undici-types": { + "version": "7.18.2", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.18.2.tgz", + "integrity": "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w==", + "license": "MIT" + }, + "node_modules/ws": { + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.20.0.tgz", + "integrity": "sha512-sAt8BhgNbzCtgGbt2OxmpuryO63ZoDk/sqaB/znQm94T4fCEsy/yV+7CdC1kJhOU9lboAEU7R3kquuycDoibVA==", + "license": "MIT", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, + "node_modules/youtubei.js": { + "version": "17.0.1", + "resolved": "https://registry.npmjs.org/youtubei.js/-/youtubei.js-17.0.1.tgz", + "integrity": "sha512-1lO4b8UqMDzE0oh2qEGzbBOd4UYRdxn/4PdpRM7BGTHxM6ddsEsKZTu90jp8V9FHVgC2h1UirQyqoqLiKwl+Zg==", + "funding": [ + "https://github.com/sponsors/LuanRT" + ], + "license": "MIT", + "dependencies": { + "@bufbuild/protobuf": "^2.0.0", + "meriyah": "^6.1.4" + } + } + } +} diff --git a/youtube-worker/package.json b/youtube-worker/package.json new file mode 100644 index 0000000..f474e9d --- /dev/null +++ b/youtube-worker/package.json @@ -0,0 +1,13 @@ +{ + "name": "youtube-worker", + "version": "1.0.0", + "private": true, + "type": "module", + "scripts": { + "start": "node server.js" + }, + "dependencies": { + "youtubei.js": "^17.0.1", + "@supabase/supabase-js": "^2.49.1" + } +} diff --git a/youtube-worker/server.js b/youtube-worker/server.js new file mode 100644 index 0000000..5455a31 --- /dev/null +++ b/youtube-worker/server.js @@ -0,0 +1,218 @@ +import http from "node:http"; +import { execFile } from "node:child_process"; +import { readFile, unlink, mkdir } from "node:fs/promises"; +import { promisify } from "node:util"; +import { Innertube } from "youtubei.js"; +import { createClient } from "@supabase/supabase-js"; + +const execFileAsync = promisify(execFile); + +// ── Config ──────────────────────────────────────────────────── +const PORT = parseInt(process.env.PORT || "3001", 10); +const SUPABASE_URL = process.env.SUPABASE_URL; +const SUPABASE_SERVICE_ROLE_KEY = process.env.SUPABASE_SERVICE_ROLE_KEY; +const STORAGE_BUCKET = process.env.STORAGE_BUCKET || "workout-audio"; +// Public-facing URL for constructing browser-accessible storage URLs. +// SUPABASE_URL is the internal Docker network URL (e.g. http://kong:8000) +// which browsers cannot reach. +const SUPABASE_PUBLIC_URL = process.env.SUPABASE_PUBLIC_URL || SUPABASE_URL; + +if (!SUPABASE_URL || !SUPABASE_SERVICE_ROLE_KEY) { + console.error("Missing SUPABASE_URL or SUPABASE_SERVICE_ROLE_KEY"); + process.exit(1); +} + +const supabase = createClient(SUPABASE_URL, SUPABASE_SERVICE_ROLE_KEY); + +// ── YouTube client (singleton, reused across requests) ──────── +let ytClient = null; +let ytClientPromise = null; + +async function getYouTubeClient() { + if (ytClient) return ytClient; + if (ytClientPromise) return ytClientPromise; + + ytClientPromise = Innertube.create({ + generate_session_locally: true, + retrieve_player: true, + }).then((client) => { + ytClient = client; + ytClientPromise = null; + console.log("YouTube client initialized"); + return client; + }); + + return ytClientPromise; +} + +// Pre-warm the client on startup +getYouTubeClient().catch((err) => + console.error("Failed to pre-warm YouTube client:", err.message) +); + +// ── Helpers ─────────────────────────────────────────────────── + +function parsePlaylistId(url) { + if (/^[A-Za-z0-9_-]+$/.test(url) && !url.includes(".")) { + return url; + } + try { + const parsed = new URL(url); + const listParam = parsed.searchParams.get("list"); + if (listParam) return listParam; + } catch {} + throw new Error(`Could not parse playlist ID from: "${url}"`); +} + +function readBody(req) { + return new Promise((resolve, reject) => { + const chunks = []; + req.on("data", (c) => chunks.push(c)); + req.on("end", () => { + try { + resolve(JSON.parse(Buffer.concat(chunks).toString())); + } catch (e) { + reject(new Error("Invalid JSON body")); + } + }); + req.on("error", reject); + }); +} + +function jsonResponse(res, status, data) { + res.writeHead(status, { "Content-Type": "application/json" }); + res.end(JSON.stringify(data)); +} + +// ── Routes ──────────────────────────────────────────────────── + +async function handlePlaylist(req, res) { + const { playlistUrl } = await readBody(req); + if (!playlistUrl) { + return jsonResponse(res, 400, { error: "playlistUrl is required" }); + } + + const playlistId = parsePlaylistId(playlistUrl); + const yt = await getYouTubeClient(); + const playlist = await yt.getPlaylist(playlistId); + + const title = playlist.info.title ?? "Untitled Playlist"; + const items = []; + + function extractItems(page) { + if (!page.items) return; + for (const item of page.items) { + // Only include items that have a video ID (skip Shorts/Reels) + if (!item.id) continue; + items.push({ + videoId: item.id, + title: item.title?.toString() ?? "Untitled", + durationSeconds: item.duration?.seconds ?? 0, + thumbnailUrl: item.thumbnails?.[0]?.url ?? null, + }); + } + } + + // First page + extractItems(playlist); + + // Pagination + let page = playlist; + while (page.has_continuation) { + page = await page.getContinuation(); + extractItems(page); + } + + console.log(`Playlist "${title}": ${items.length} items`); + + jsonResponse(res, 200, { title, items }); +} + +async function handleDownload(req, res) { + const { videoId, jobId } = await readBody(req); + if (!videoId || !jobId) { + return jsonResponse(res, 400, { error: "videoId and jobId are required" }); + } + + // Use yt-dlp for the actual download — it handles PO tokens, signature + // deciphering, and anti-bot measures that youtubei.js cannot. + const tmpDir = `/tmp/ytdl-${jobId}`; + const outPath = `${tmpDir}/${videoId}.m4a`; + + await mkdir(tmpDir, { recursive: true }); + + try { + console.log(`Downloading ${videoId} via yt-dlp...`); + + const { stderr } = await execFileAsync("yt-dlp", [ + "-f", "ba[ext=m4a]/ba", // best audio in m4a, fallback to best audio + "--no-playlist", + "--no-warnings", + "-x", // extract audio + "--audio-format", "m4a", // ensure m4a output + "--audio-quality", "0", // best quality + "-o", outPath, + `https://www.youtube.com/watch?v=${videoId}`, + ], { timeout: 120_000 }); + + if (stderr) console.warn(`yt-dlp stderr for ${videoId}: ${stderr}`); + + // Read the downloaded file + const audioData = await readFile(outPath); + console.log( + `Downloaded ${videoId}: ${(audioData.length / 1024 / 1024).toFixed(1)} MB` + ); + + // Upload to Supabase Storage — flat structure, no subfolder + const storagePath = `${videoId}.m4a`; + + const { error: uploadError } = await supabase.storage + .from(STORAGE_BUCKET) + .upload(storagePath, audioData, { + contentType: "audio/mp4", + cacheControl: "3600", + upsert: true, + }); + + if (uploadError) { + throw new Error(`Storage upload failed: ${uploadError.message}`); + } + + // Construct public URL using the external-facing base URL, not the + // internal Docker URL that supabase.storage.getPublicUrl() would use. + const publicUrl = `${SUPABASE_PUBLIC_URL}/storage/v1/object/public/${STORAGE_BUCKET}/${storagePath}`; + + jsonResponse(res, 200, { storagePath, publicUrl }); + } finally { + // Clean up temp file + await unlink(outPath).catch(() => {}); + } +} + +async function handleHealth(_req, res) { + jsonResponse(res, 200, { status: "ok" }); +} + +// ── Server ──────────────────────────────────────────────────── + +const server = http.createServer(async (req, res) => { + try { + if (req.method === "GET" && req.url === "/health") { + return await handleHealth(req, res); + } + if (req.method === "POST" && req.url === "/playlist") { + return await handlePlaylist(req, res); + } + if (req.method === "POST" && req.url === "/download") { + return await handleDownload(req, res); + } + jsonResponse(res, 404, { error: "Not found" }); + } catch (err) { + console.error(`${req.method} ${req.url} error:`, err.message); + jsonResponse(res, 500, { error: err.message }); + } +}); + +server.listen(PORT, () => { + console.log(`youtube-worker listening on port ${PORT}`); +});