feat: YouTube music download system with admin dashboard
Sidecar architecture: edge functions (auth + DB) → youtube-worker container (youtubei.js for playlist metadata, yt-dlp for audio downloads). - Edge functions: youtube-playlist, youtube-process, youtube-status (CRUD) - youtube-worker sidecar: Node.js + yt-dlp, downloads M4A to Supabase Storage - Admin music page: import playlists, process queue, inline genre editing - 12 music genres with per-track assignment and import-time defaults - DB migrations: download_jobs, download_items tables with genre column - Deploy script and CI workflow for edge functions + sidecar
This commit is contained in:
Millian Lamiaux
21
youtube-worker/Dockerfile
Normal file
21
youtube-worker/Dockerfile
Normal file
@@ -0,0 +1,21 @@
|
||||
FROM node:20-slim
|
||||
|
||||
# Install yt-dlp (standalone binary) and ffmpeg for audio extraction
|
||||
RUN apt-get update && \
|
||||
apt-get install -y --no-install-recommends python3 ffmpeg curl ca-certificates && \
|
||||
curl -L https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp -o /usr/local/bin/yt-dlp && \
|
||||
chmod +x /usr/local/bin/yt-dlp && \
|
||||
apt-get remove -y curl && \
|
||||
apt-get autoremove -y && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY package.json package-lock.json* ./
|
||||
RUN npm install --production
|
||||
|
||||
COPY server.js ./
|
||||
|
||||
EXPOSE 3001
|
||||
|
||||
CMD ["node", "server.js"]
|
||||
190
youtube-worker/package-lock.json
generated
Normal file
190
youtube-worker/package-lock.json
generated
Normal file
@@ -0,0 +1,190 @@
|
||||
{
|
||||
"name": "youtube-worker",
|
||||
"version": "1.0.0",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "youtube-worker",
|
||||
"version": "1.0.0",
|
||||
"dependencies": {
|
||||
"@supabase/supabase-js": "^2.49.1",
|
||||
"youtubei.js": "^17.0.1"
|
||||
}
|
||||
},
|
||||
"node_modules/@bufbuild/protobuf": {
|
||||
"version": "2.11.0",
|
||||
"resolved": "https://registry.npmjs.org/@bufbuild/protobuf/-/protobuf-2.11.0.tgz",
|
||||
"integrity": "sha512-sBXGT13cpmPR5BMgHE6UEEfEaShh5Ror6rfN3yEK5si7QVrtZg8LEPQb0VVhiLRUslD2yLnXtnRzG035J/mZXQ==",
|
||||
"license": "(Apache-2.0 AND BSD-3-Clause)"
|
||||
},
|
||||
"node_modules/@supabase/auth-js": {
|
||||
"version": "2.100.0",
|
||||
"resolved": "https://registry.npmjs.org/@supabase/auth-js/-/auth-js-2.100.0.tgz",
|
||||
"integrity": "sha512-pdT3ye3UVRN1Cg0wom6BmyY+XTtp5DiJaYnPi6j8ht5i8Lq8kfqxJMJz9GI9YDKk3w1nhGOPnh6Qz5qpyYm+1w==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"tslib": "2.8.1"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=20.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@supabase/functions-js": {
|
||||
"version": "2.100.0",
|
||||
"resolved": "https://registry.npmjs.org/@supabase/functions-js/-/functions-js-2.100.0.tgz",
|
||||
"integrity": "sha512-keLg79RPwP+uiwHuxFPTFgDRxPV46LM4j/swjyR2GKJgWniTVSsgiBHfbIBDcrQwehLepy09b/9QSHUywtKRWQ==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"tslib": "2.8.1"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=20.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@supabase/phoenix": {
|
||||
"version": "0.4.0",
|
||||
"resolved": "https://registry.npmjs.org/@supabase/phoenix/-/phoenix-0.4.0.tgz",
|
||||
"integrity": "sha512-RHSx8bHS02xwfHdAbX5Lpbo6PXbgyf7lTaXTlwtFDPwOIw64NnVRwFAXGojHhjtVYI+PEPNSWwkL90f4agN3bw==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@supabase/postgrest-js": {
|
||||
"version": "2.100.0",
|
||||
"resolved": "https://registry.npmjs.org/@supabase/postgrest-js/-/postgrest-js-2.100.0.tgz",
|
||||
"integrity": "sha512-xYNvNbBJaXOGcrZ44wxwp5830uo1okMHGS8h8dm3u4f0xcZ39yzbryUsubTJW41MG2gbL/6U57cA4Pi6YMZ9pA==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"tslib": "2.8.1"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=20.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@supabase/realtime-js": {
|
||||
"version": "2.100.0",
|
||||
"resolved": "https://registry.npmjs.org/@supabase/realtime-js/-/realtime-js-2.100.0.tgz",
|
||||
"integrity": "sha512-2AZs00zzEF0HuCKY8grz5eCYlwEfVi5HONLZFoNR6aDfxQivl8zdQYNjyFoqN2MZiVhQHD7u6XV/xHwM8mCEHw==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@supabase/phoenix": "^0.4.0",
|
||||
"@types/ws": "^8.18.1",
|
||||
"tslib": "2.8.1",
|
||||
"ws": "^8.18.2"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=20.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@supabase/storage-js": {
|
||||
"version": "2.100.0",
|
||||
"resolved": "https://registry.npmjs.org/@supabase/storage-js/-/storage-js-2.100.0.tgz",
|
||||
"integrity": "sha512-d4EeuK6RNIgYNA2MU9kj8lQrLm5AzZ+WwpWjGkii6SADQNIGTC/uiaTRu02XJ5AmFALQfo8fLl9xuCkO6Xw+iQ==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"iceberg-js": "^0.8.1",
|
||||
"tslib": "2.8.1"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=20.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@supabase/supabase-js": {
|
||||
"version": "2.100.0",
|
||||
"resolved": "https://registry.npmjs.org/@supabase/supabase-js/-/supabase-js-2.100.0.tgz",
|
||||
"integrity": "sha512-r0tlcukejJXJ1m/2eG/Ya5eYs4W8AC7oZfShpG3+SIo/eIU9uIt76ZeYI1SoUwUmcmzlAbgch+HDZDR/toVQPQ==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@supabase/auth-js": "2.100.0",
|
||||
"@supabase/functions-js": "2.100.0",
|
||||
"@supabase/postgrest-js": "2.100.0",
|
||||
"@supabase/realtime-js": "2.100.0",
|
||||
"@supabase/storage-js": "2.100.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=20.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/node": {
|
||||
"version": "25.5.0",
|
||||
"resolved": "https://registry.npmjs.org/@types/node/-/node-25.5.0.tgz",
|
||||
"integrity": "sha512-jp2P3tQMSxWugkCUKLRPVUpGaL5MVFwF8RDuSRztfwgN1wmqJeMSbKlnEtQqU8UrhTmzEmZdu2I6v2dpp7XIxw==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"undici-types": "~7.18.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/ws": {
|
||||
"version": "8.18.1",
|
||||
"resolved": "https://registry.npmjs.org/@types/ws/-/ws-8.18.1.tgz",
|
||||
"integrity": "sha512-ThVF6DCVhA8kUGy+aazFQ4kXQ7E1Ty7A3ypFOe0IcJV8O/M511G99AW24irKrW56Wt44yG9+ij8FaqoBGkuBXg==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@types/node": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/iceberg-js": {
|
||||
"version": "0.8.1",
|
||||
"resolved": "https://registry.npmjs.org/iceberg-js/-/iceberg-js-0.8.1.tgz",
|
||||
"integrity": "sha512-1dhVQZXhcHje7798IVM+xoo/1ZdVfzOMIc8/rgVSijRK38EDqOJoGula9N/8ZI5RD8QTxNQtK/Gozpr+qUqRRA==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=20.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/meriyah": {
|
||||
"version": "6.1.4",
|
||||
"resolved": "https://registry.npmjs.org/meriyah/-/meriyah-6.1.4.tgz",
|
||||
"integrity": "sha512-Sz8FzjzI0kN13GK/6MVEsVzMZEPvOhnmmI1lU5+/1cGOiK3QUahntrNNtdVeihrO7t9JpoH75iMNXg6R6uWflQ==",
|
||||
"license": "ISC",
|
||||
"engines": {
|
||||
"node": ">=18.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/tslib": {
|
||||
"version": "2.8.1",
|
||||
"resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz",
|
||||
"integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==",
|
||||
"license": "0BSD"
|
||||
},
|
||||
"node_modules/undici-types": {
|
||||
"version": "7.18.2",
|
||||
"resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.18.2.tgz",
|
||||
"integrity": "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/ws": {
|
||||
"version": "8.20.0",
|
||||
"resolved": "https://registry.npmjs.org/ws/-/ws-8.20.0.tgz",
|
||||
"integrity": "sha512-sAt8BhgNbzCtgGbt2OxmpuryO63ZoDk/sqaB/znQm94T4fCEsy/yV+7CdC1kJhOU9lboAEU7R3kquuycDoibVA==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=10.0.0"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"bufferutil": "^4.0.1",
|
||||
"utf-8-validate": ">=5.0.2"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
"bufferutil": {
|
||||
"optional": true
|
||||
},
|
||||
"utf-8-validate": {
|
||||
"optional": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/youtubei.js": {
|
||||
"version": "17.0.1",
|
||||
"resolved": "https://registry.npmjs.org/youtubei.js/-/youtubei.js-17.0.1.tgz",
|
||||
"integrity": "sha512-1lO4b8UqMDzE0oh2qEGzbBOd4UYRdxn/4PdpRM7BGTHxM6ddsEsKZTu90jp8V9FHVgC2h1UirQyqoqLiKwl+Zg==",
|
||||
"funding": [
|
||||
"https://github.com/sponsors/LuanRT"
|
||||
],
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@bufbuild/protobuf": "^2.0.0",
|
||||
"meriyah": "^6.1.4"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
13
youtube-worker/package.json
Normal file
13
youtube-worker/package.json
Normal file
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"name": "youtube-worker",
|
||||
"version": "1.0.0",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"start": "node server.js"
|
||||
},
|
||||
"dependencies": {
|
||||
"youtubei.js": "^17.0.1",
|
||||
"@supabase/supabase-js": "^2.49.1"
|
||||
}
|
||||
}
|
||||
218
youtube-worker/server.js
Normal file
218
youtube-worker/server.js
Normal file
@@ -0,0 +1,218 @@
|
||||
import http from "node:http";
|
||||
import { execFile } from "node:child_process";
|
||||
import { readFile, unlink, mkdir } from "node:fs/promises";
|
||||
import { promisify } from "node:util";
|
||||
import { Innertube } from "youtubei.js";
|
||||
import { createClient } from "@supabase/supabase-js";
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
|
||||
// ── Config ────────────────────────────────────────────────────
|
||||
const PORT = parseInt(process.env.PORT || "3001", 10);
|
||||
const SUPABASE_URL = process.env.SUPABASE_URL;
|
||||
const SUPABASE_SERVICE_ROLE_KEY = process.env.SUPABASE_SERVICE_ROLE_KEY;
|
||||
const STORAGE_BUCKET = process.env.STORAGE_BUCKET || "workout-audio";
|
||||
// Public-facing URL for constructing browser-accessible storage URLs.
|
||||
// SUPABASE_URL is the internal Docker network URL (e.g. http://kong:8000)
|
||||
// which browsers cannot reach.
|
||||
const SUPABASE_PUBLIC_URL = process.env.SUPABASE_PUBLIC_URL || SUPABASE_URL;
|
||||
|
||||
if (!SUPABASE_URL || !SUPABASE_SERVICE_ROLE_KEY) {
|
||||
console.error("Missing SUPABASE_URL or SUPABASE_SERVICE_ROLE_KEY");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const supabase = createClient(SUPABASE_URL, SUPABASE_SERVICE_ROLE_KEY);
|
||||
|
||||
// ── YouTube client (singleton, reused across requests) ────────
|
||||
let ytClient = null;
|
||||
let ytClientPromise = null;
|
||||
|
||||
async function getYouTubeClient() {
|
||||
if (ytClient) return ytClient;
|
||||
if (ytClientPromise) return ytClientPromise;
|
||||
|
||||
ytClientPromise = Innertube.create({
|
||||
generate_session_locally: true,
|
||||
retrieve_player: true,
|
||||
}).then((client) => {
|
||||
ytClient = client;
|
||||
ytClientPromise = null;
|
||||
console.log("YouTube client initialized");
|
||||
return client;
|
||||
});
|
||||
|
||||
return ytClientPromise;
|
||||
}
|
||||
|
||||
// Pre-warm the client on startup
|
||||
getYouTubeClient().catch((err) =>
|
||||
console.error("Failed to pre-warm YouTube client:", err.message)
|
||||
);
|
||||
|
||||
// ── Helpers ───────────────────────────────────────────────────
|
||||
|
||||
function parsePlaylistId(url) {
|
||||
if (/^[A-Za-z0-9_-]+$/.test(url) && !url.includes(".")) {
|
||||
return url;
|
||||
}
|
||||
try {
|
||||
const parsed = new URL(url);
|
||||
const listParam = parsed.searchParams.get("list");
|
||||
if (listParam) return listParam;
|
||||
} catch {}
|
||||
throw new Error(`Could not parse playlist ID from: "${url}"`);
|
||||
}
|
||||
|
||||
function readBody(req) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const chunks = [];
|
||||
req.on("data", (c) => chunks.push(c));
|
||||
req.on("end", () => {
|
||||
try {
|
||||
resolve(JSON.parse(Buffer.concat(chunks).toString()));
|
||||
} catch (e) {
|
||||
reject(new Error("Invalid JSON body"));
|
||||
}
|
||||
});
|
||||
req.on("error", reject);
|
||||
});
|
||||
}
|
||||
|
||||
function jsonResponse(res, status, data) {
|
||||
res.writeHead(status, { "Content-Type": "application/json" });
|
||||
res.end(JSON.stringify(data));
|
||||
}
|
||||
|
||||
// ── Routes ────────────────────────────────────────────────────
|
||||
|
||||
async function handlePlaylist(req, res) {
|
||||
const { playlistUrl } = await readBody(req);
|
||||
if (!playlistUrl) {
|
||||
return jsonResponse(res, 400, { error: "playlistUrl is required" });
|
||||
}
|
||||
|
||||
const playlistId = parsePlaylistId(playlistUrl);
|
||||
const yt = await getYouTubeClient();
|
||||
const playlist = await yt.getPlaylist(playlistId);
|
||||
|
||||
const title = playlist.info.title ?? "Untitled Playlist";
|
||||
const items = [];
|
||||
|
||||
function extractItems(page) {
|
||||
if (!page.items) return;
|
||||
for (const item of page.items) {
|
||||
// Only include items that have a video ID (skip Shorts/Reels)
|
||||
if (!item.id) continue;
|
||||
items.push({
|
||||
videoId: item.id,
|
||||
title: item.title?.toString() ?? "Untitled",
|
||||
durationSeconds: item.duration?.seconds ?? 0,
|
||||
thumbnailUrl: item.thumbnails?.[0]?.url ?? null,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// First page
|
||||
extractItems(playlist);
|
||||
|
||||
// Pagination
|
||||
let page = playlist;
|
||||
while (page.has_continuation) {
|
||||
page = await page.getContinuation();
|
||||
extractItems(page);
|
||||
}
|
||||
|
||||
console.log(`Playlist "${title}": ${items.length} items`);
|
||||
|
||||
jsonResponse(res, 200, { title, items });
|
||||
}
|
||||
|
||||
async function handleDownload(req, res) {
|
||||
const { videoId, jobId } = await readBody(req);
|
||||
if (!videoId || !jobId) {
|
||||
return jsonResponse(res, 400, { error: "videoId and jobId are required" });
|
||||
}
|
||||
|
||||
// Use yt-dlp for the actual download — it handles PO tokens, signature
|
||||
// deciphering, and anti-bot measures that youtubei.js cannot.
|
||||
const tmpDir = `/tmp/ytdl-${jobId}`;
|
||||
const outPath = `${tmpDir}/${videoId}.m4a`;
|
||||
|
||||
await mkdir(tmpDir, { recursive: true });
|
||||
|
||||
try {
|
||||
console.log(`Downloading ${videoId} via yt-dlp...`);
|
||||
|
||||
const { stderr } = await execFileAsync("yt-dlp", [
|
||||
"-f", "ba[ext=m4a]/ba", // best audio in m4a, fallback to best audio
|
||||
"--no-playlist",
|
||||
"--no-warnings",
|
||||
"-x", // extract audio
|
||||
"--audio-format", "m4a", // ensure m4a output
|
||||
"--audio-quality", "0", // best quality
|
||||
"-o", outPath,
|
||||
`https://www.youtube.com/watch?v=${videoId}`,
|
||||
], { timeout: 120_000 });
|
||||
|
||||
if (stderr) console.warn(`yt-dlp stderr for ${videoId}: ${stderr}`);
|
||||
|
||||
// Read the downloaded file
|
||||
const audioData = await readFile(outPath);
|
||||
console.log(
|
||||
`Downloaded ${videoId}: ${(audioData.length / 1024 / 1024).toFixed(1)} MB`
|
||||
);
|
||||
|
||||
// Upload to Supabase Storage — flat structure, no subfolder
|
||||
const storagePath = `${videoId}.m4a`;
|
||||
|
||||
const { error: uploadError } = await supabase.storage
|
||||
.from(STORAGE_BUCKET)
|
||||
.upload(storagePath, audioData, {
|
||||
contentType: "audio/mp4",
|
||||
cacheControl: "3600",
|
||||
upsert: true,
|
||||
});
|
||||
|
||||
if (uploadError) {
|
||||
throw new Error(`Storage upload failed: ${uploadError.message}`);
|
||||
}
|
||||
|
||||
// Construct public URL using the external-facing base URL, not the
|
||||
// internal Docker URL that supabase.storage.getPublicUrl() would use.
|
||||
const publicUrl = `${SUPABASE_PUBLIC_URL}/storage/v1/object/public/${STORAGE_BUCKET}/${storagePath}`;
|
||||
|
||||
jsonResponse(res, 200, { storagePath, publicUrl });
|
||||
} finally {
|
||||
// Clean up temp file
|
||||
await unlink(outPath).catch(() => {});
|
||||
}
|
||||
}
|
||||
|
||||
async function handleHealth(_req, res) {
|
||||
jsonResponse(res, 200, { status: "ok" });
|
||||
}
|
||||
|
||||
// ── Server ────────────────────────────────────────────────────
|
||||
|
||||
const server = http.createServer(async (req, res) => {
|
||||
try {
|
||||
if (req.method === "GET" && req.url === "/health") {
|
||||
return await handleHealth(req, res);
|
||||
}
|
||||
if (req.method === "POST" && req.url === "/playlist") {
|
||||
return await handlePlaylist(req, res);
|
||||
}
|
||||
if (req.method === "POST" && req.url === "/download") {
|
||||
return await handleDownload(req, res);
|
||||
}
|
||||
jsonResponse(res, 404, { error: "Not found" });
|
||||
} catch (err) {
|
||||
console.error(`${req.method} ${req.url} error:`, err.message);
|
||||
jsonResponse(res, 500, { error: err.message });
|
||||
}
|
||||
});
|
||||
|
||||
server.listen(PORT, () => {
|
||||
console.log(`youtube-worker listening on port ${PORT}`);
|
||||
});
|
||||
Reference in New Issue
Block a user