Skip to content
Merged
Show file tree
Hide file tree
Changes from 99 commits
Commits
Show all changes
102 commits
Select commit Hold shift + click to select a range
dbb83b9
web/i18n/settings: remove unused strings
wukko Jun 11, 2025
eb90843
web/pagenav: use pop() instead of at(-1)
dumbmoron Jun 11, 2025
a06baa4
web: add uuid() function with fallback if randomUUID is missing
dumbmoron Jun 11, 2025
81c8daf
web/storage: robuster er opfs availability check
dumbmoron Jun 11, 2025
d0298db
web/i18n/dialog: remove unused strings
wukko Jun 12, 2025
ace654e
web/i18n/dialog: remove even more unused strings
wukko Jun 12, 2025
863d39d
web/i18n/about: remove an unused string
wukko Jun 12, 2025
5ea170a
web: deprecate youtube HLS, enable it only via env variable
wukko Jun 14, 2025
507fab8
web/workers/ffmpeg: proper error code for missing audio channel error
wukko Jun 14, 2025
e18575f
web/i18n/error/api: update youtube.no_matching_format
wukko Jun 15, 2025
2ac9153
web/CaptchaTooltip: increase max width
wukko Jun 15, 2025
5e7f9c5
api/package: update youtubei.js to 14.0.0
wukko Jun 16, 2025
1e7406d
web/i18n/error/api: rephrase youtube.no_matching_format
wukko Jun 16, 2025
af99e72
api: disable youtube HLS by default & add env to enable it
wukko Jun 17, 2025
b880157
api/env: SESSION_RATELIMIT -> SESSION_RATELIMIT_MAX
wukko Jun 17, 2025
19be25c
docs/api-env-variables: fix ratelimit variable names
wukko Jun 17, 2025
c0d3c21
docs/api-env-variables: add info about ENABLE_DEPRECATED_YOUTUBE_HLS
wukko Jun 17, 2025
1e35931
docs/api-env-variables: update FORCE_LOCAL_PROCESSING section
wukko Jun 17, 2025
967552b
api/schema: add subtitleLang
wukko Jun 18, 2025
259a075
api: initial subtitles functionality with youtube support
wukko Jun 18, 2025
b91c0c0
api/stream/types: specify subtitle format for containers other than mp4
wukko Jun 19, 2025
672b3dc
api/match-action: convert ISO 639-1 language codes to ISO 639-2
wukko Jun 19, 2025
fef6ee1
web/i18n/error/queue: add missing generic_error
wukko Jun 19, 2025
4da95e0
web/libav: disable wasm multithreading on old ios
wukko Jun 19, 2025
2396462
api/schema: add youtubeVideoContainer
wukko Jun 20, 2025
eb249a3
api/match: ignore subtitleLang if it's "none"
wukko Jun 20, 2025
33c801f
api/youtube: add support for youtubeVideoContainer
wukko Jun 20, 2025
c4e910d
api/stream/types: refactor, support mkv, don't duplicate args
wukko Jun 20, 2025
3daf1c4
web: refactor youtube-lang
wukko Jun 20, 2025
6e394cd
web/settings: add youtubeVideoContainer & subtitleLang
wukko Jun 20, 2025
c9fdfca
web/SettingsDropdown: prevent crash if selectedTitle is undefined
wukko Jun 20, 2025
1e5cc35
web/audio-sub-language: refactor, prioritize popular languages
wukko Jun 20, 2025
5860c50
web/settings/video: add youtube container settings
wukko Jun 20, 2025
a30a27a
web/settings/metadata: add subtitles language dropdown
wukko Jun 20, 2025
9f7f637
web/api/saving-handler: add youtubeVideoContainer & subtitleLang
wukko Jun 20, 2025
993a885
web/util: add support for subtitle track language metadata
wukko Jun 20, 2025
7ce9d68
api/youtube: don't use session if user wants subtitles
wukko Jun 20, 2025
96a02d5
web/package: update libav packages
wukko Jun 20, 2025
0b0f0d6
web/queue: add subtitle codec args
wukko Jun 20, 2025
337edfc
api/request/local-processing: return subtitles boolean
wukko Jun 20, 2025
254ad96
web/queue: add subtitle args when output has subtitles
wukko Jun 20, 2025
a5838f3
api/stream/types: add subtitles & metadata to remux
wukko Jun 20, 2025
a44bea6
api/vimeo: add subtitle parsing from the mobile api
wukko Jun 20, 2025
17ab8dd
web/queue: add subtitles independently from remux type
wukko Jun 20, 2025
ab526c2
api/loom: add transcription subtitles
wukko Jun 20, 2025
d18b22e
api/processing/request: return a unique error code
wukko Jun 20, 2025
aff2d22
api/language-codes: add reverse lookup (2 to 1)
wukko Jun 20, 2025
630e4a6
api/tiktok: add support for subtitles
wukko Jun 20, 2025
2c0a1b6
web/i18n/settings: update subtitles description
wukko Jun 20, 2025
a998a57
web/queue: refactor media icon selection
wukko Jun 22, 2025
a6b599a
api/schema: transform localProcessing to enum
wukko Jun 22, 2025
28ab274
api/match-action: support forced local processing
wukko Jun 22, 2025
ac85ce8
api/processing/request: backwards compat with boolean localProcessing
wukko Jun 22, 2025
a4d5f5b
web/settings: migrate boolean localProcessing to enum
wukko Jun 22, 2025
8853989
web/settings/local: transform the media processing setting to a switcher
wukko Jun 22, 2025
61e0862
web/types/api: add proxy local processing type
wukko Jun 22, 2025
f883887
web/queue: don't try to add a remux task if response type is proxy
wukko Jun 22, 2025
05fb160
api/match: update forcing local processing via env
wukko Jun 22, 2025
0fca9c4
api/schema: remove deprecated variables
wukko Jun 22, 2025
21c4a1e
api/match: set alwaysProxy to true if local processing is forced
wukko Jun 22, 2025
6d62bce
api/match-action: don't force local-processing response for pickers
wukko Jun 22, 2025
b384dc8
web/error/api: add missing "the" to fetch.critical.core
wukko Jun 22, 2025
599ec9d
web/UpdateNotification: update margin & font size
wukko Jun 22, 2025
44f4ea3
api/stream/internal: stream vk videos in chunks
wukko Jun 24, 2025
997b06e
api/vk: add support for subtitles
wukko Jun 24, 2025
ff06a10
api/processing/url: improve vk url parsing
wukko Jun 24, 2025
75691d4
api/tests/facebook: replace a dead link
wukko Jun 24, 2025
28b8538
api/vk: allow auto generated subs & pick explicitly vtt
wukko Jun 24, 2025
aa376d7
api/stream/types: huge refactor & simplification of code
wukko Jun 24, 2025
14657e5
api/stream: split types.js into proxy.js and ffmpeg.js
wukko Jun 24, 2025
4f4478a
api/ffmpeg: fix audio codec args in remux()
wukko Jun 24, 2025
d3793c7
api/ffmpeg: map video and audio in remux() with one main input
wukko Jun 24, 2025
fcdf5da
api/ffmpeg: refactor even more
wukko Jun 25, 2025
52695cb
api/service-config: replace static arrays with sets
wukko Jun 25, 2025
3dae5b2
api/ffmpeg: move stream type + url count check to remux()
wukko Jun 25, 2025
f4637b7
api/rutube: add subtitles
wukko Jun 25, 2025
f7e5951
web/lib/device: enable local processing on all ios devices
wukko Jun 25, 2025
4ff4766
docs/api: add info about subtitle bool in local processing response
wukko Jun 26, 2025
164ea8a
api: return covers from soundcloud and youtube
wukko Jun 26, 2025
e4ce873
web/queue: add audio covers & crop them when needed
wukko Jun 26, 2025
655e7a5
docs/api: add info about cover & cropCover
wukko Jun 26, 2025
84aa80e
api/match-action: don't add cover if metadata is disabled
wukko Jun 26, 2025
bfb23c8
web/queue: add cover only to mp3 files
wukko Jun 26, 2025
81a0d5e
web/queue: scale cropped covers to 720x720
wukko Jun 26, 2025
d69100c
api/tiktok: validate that redirected link is still tiktok
wukko Jun 26, 2025
3243564
api/api-keys: add allowedServices to limit or extend access
wukko Jun 26, 2025
dce9eb3
docs/protect-an-instance: add info about allowedServices in api keys
wukko Jun 26, 2025
8feaf5c
api/api-keys: replace .find() with .some() in allowedServices
wukko Jun 26, 2025
900c6f2
api/tests/vimeo: allow mature video tests to fail
wukko Jun 27, 2025
51c5d05
api/service-patterns/tiktok: allow longer shortLink
wukko Jun 28, 2025
fffb31d
web/i18n/error/api: fix a typo in fetch.short_link
wukko Jun 28, 2025
9d81830
api/twitter: add subtitle extraction
wukko Jun 28, 2025
7298082
api: refactor two static arrays to set
wukko Jun 28, 2025
c164441
api/env: backwards compatibility with SESSION_RATELIMIT
wukko Jun 28, 2025
3d2473d
web/audio-sub-language: refactor to avoid code duplication
wukko Jun 28, 2025
bc8c16f
web/env: accept 1 as bool value
wukko Jun 28, 2025
d70180b
api/core: merge isApiKey and isSession into authType
wukko Jun 28, 2025
4fc2952
web/audio-sub-language: update localized values dynamically
wukko Jun 28, 2025
bd0caac
web/changelogs/11.0: set a fixed commit in compare, fix env name error
wukko Jun 28, 2025
a751f81
version-info: return git branch info correctly in cf workers
wukko Jun 28, 2025
8da71e4
api/package: bump version to 11.2
wukko Jun 28, 2025
a60e94d
web/package: bump version to 11.2
wukko Jun 28, 2025
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion api/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@
"set-cookie-parser": "2.6.0",
"undici": "^5.19.1",
"url-pattern": "1.0.3",
"youtubei.js": "^13.4.0",
"youtubei.js": "^14.0.0",
"zod": "^3.23.8"
},
"optionalDependencies": {
Expand Down
10 changes: 7 additions & 3 deletions api/src/core/api.js
Original file line number Diff line number Diff line change
Expand Up @@ -156,6 +156,7 @@ export const runAPI = async (express, app, __dirname, isPrimary = true) => {
return fail(res, `error.api.auth.key.${error}`);
}

req.authType = "key";
return next();
});

Expand Down Expand Up @@ -184,7 +185,7 @@ export const runAPI = async (express, app, __dirname, isPrimary = true) => {
}

req.rateLimitKey = hashHmac(token, 'rate');
req.isSession = true;
req.authType = "session";
} catch {
return fail(res, "error.api.generic");
}
Expand Down Expand Up @@ -244,7 +245,10 @@ export const runAPI = async (express, app, __dirname, isPrimary = true) => {
return fail(res, "error.api.invalid_body");
}

const parsed = extract(normalizedRequest.url);
const parsed = extract(
normalizedRequest.url,
APIKeys.getAllowedServices(req.rateLimitKey),
);

if (!parsed) {
return fail(res, "error.api.link.invalid");
Expand All @@ -263,7 +267,7 @@ export const runAPI = async (express, app, __dirname, isPrimary = true) => {
host: parsed.host,
patternMatch: parsed.patternMatch,
params: normalizedRequest,
isSession: req.isSession ?? false,
authType: req.authType ?? "none",
});

res.status(result.status).json(result.body);
Expand Down
19 changes: 18 additions & 1 deletion api/src/core/env.js
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,10 @@ import * as cluster from "../misc/cluster.js";
import { Green, Yellow } from "../misc/console-text.js";

const forceLocalProcessingOptions = ["never", "session", "always"];
const youtubeHlsOptions = ["never", "key", "always"];

export const loadEnvs = (env = process.env) => {
const allServices = new Set(Object.keys(services));
const disabledServices = env.DISABLED_SERVICES?.split(',') || [];
const enabledServices = new Set(Object.keys(services).filter(e => {
if (!disabledServices.includes(e)) {
Expand Down Expand Up @@ -37,7 +39,12 @@ export const loadEnvs = (env = process.env) => {
tunnelRateLimitMax: (env.TUNNEL_RATELIMIT_MAX && parseInt(env.TUNNEL_RATELIMIT_MAX)) || 40,

sessionRateLimitWindow: (env.SESSION_RATELIMIT_WINDOW && parseInt(env.SESSION_RATELIMIT_WINDOW)) || 60,
sessionRateLimit: (env.SESSION_RATELIMIT && parseInt(env.SESSION_RATELIMIT)) || 10,
sessionRateLimit:
// backwards compatibility with SESSION_RATELIMIT
// till next major due to an error in docs
(env.SESSION_RATELIMIT_MAX && parseInt(env.SESSION_RATELIMIT_MAX))
|| (env.SESSION_RATELIMIT && parseInt(env.SESSION_RATELIMIT))
|| 10,

durationLimit: (env.DURATION_LIMIT && parseInt(env.DURATION_LIMIT)) || 10800,
streamLifespan: (env.TUNNEL_LIFESPAN && parseInt(env.TUNNEL_LIFESPAN)) || 90,
Expand All @@ -63,6 +70,7 @@ export const loadEnvs = (env = process.env) => {
instanceCount: (env.API_INSTANCE_COUNT && parseInt(env.API_INSTANCE_COUNT)) || 1,
keyReloadInterval: 900,

allServices,
enabledServices,

customInnertubeClient: env.CUSTOM_INNERTUBE_CLIENT,
Expand All @@ -74,6 +82,9 @@ export const loadEnvs = (env = process.env) => {
// "never" | "session" | "always"
forceLocalProcessing: env.FORCE_LOCAL_PROCESSING ?? "never",

// "never" | "key" | "always"
enableDeprecatedYoutubeHls: env.ENABLE_DEPRECATED_YOUTUBE_HLS ?? "never",

envFile: env.API_ENV_FILE,
envRemoteReloadInterval: 300,
};
Expand Down Expand Up @@ -106,6 +117,12 @@ export const validateEnvs = async (env) => {
throw new Error("Invalid FORCE_LOCAL_PROCESSING");
}

if (env.enableDeprecatedYoutubeHls && !youtubeHlsOptions.includes(env.enableDeprecatedYoutubeHls)) {
console.error("ENABLE_DEPRECATED_YOUTUBE_HLS is invalid.");
console.error(`Supported options are are: ${youtubeHlsOptions.join(', ')}\n`);
throw new Error("Invalid ENABLE_DEPRECATED_YOUTUBE_HLS");
}

if (env.externalProxy && env.freebindCIDR) {
throw new Error('freebind is not available when external proxy is enabled')
}
Expand Down
53 changes: 53 additions & 0 deletions api/src/misc/language-codes.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
// converted from this file https://www.loc.gov/standards/iso639-2/ISO-639-2_utf-8.txt
const iso639_1to2 = {
'aa': 'aar', 'ab': 'abk', 'af': 'afr', 'ak': 'aka', 'sq': 'sqi',
'am': 'amh', 'ar': 'ara', 'an': 'arg', 'hy': 'hye', 'as': 'asm',
'av': 'ava', 'ae': 'ave', 'ay': 'aym', 'az': 'aze', 'ba': 'bak',
'bm': 'bam', 'eu': 'eus', 'be': 'bel', 'bn': 'ben', 'bi': 'bis',
'bs': 'bos', 'br': 'bre', 'bg': 'bul', 'my': 'mya', 'ca': 'cat',
'ch': 'cha', 'ce': 'che', 'zh': 'zho', 'cu': 'chu', 'cv': 'chv',
'kw': 'cor', 'co': 'cos', 'cr': 'cre', 'cs': 'ces', 'da': 'dan',
'dv': 'div', 'nl': 'nld', 'dz': 'dzo', 'en': 'eng', 'eo': 'epo',
'et': 'est', 'ee': 'ewe', 'fo': 'fao', 'fj': 'fij', 'fi': 'fin',
'fr': 'fra', 'fy': 'fry', 'ff': 'ful', 'ka': 'kat', 'de': 'deu',
'gd': 'gla', 'ga': 'gle', 'gl': 'glg', 'gv': 'glv', 'el': 'ell',
'gn': 'grn', 'gu': 'guj', 'ht': 'hat', 'ha': 'hau', 'he': 'heb',
'hz': 'her', 'hi': 'hin', 'ho': 'hmo', 'hr': 'hrv', 'hu': 'hun',
'ig': 'ibo', 'is': 'isl', 'io': 'ido', 'ii': 'iii', 'iu': 'iku',
'ie': 'ile', 'ia': 'ina', 'id': 'ind', 'ik': 'ipk', 'it': 'ita',
'jv': 'jav', 'ja': 'jpn', 'kl': 'kal', 'kn': 'kan', 'ks': 'kas',
'kr': 'kau', 'kk': 'kaz', 'km': 'khm', 'ki': 'kik', 'rw': 'kin',
'ky': 'kir', 'kv': 'kom', 'kg': 'kon', 'ko': 'kor', 'kj': 'kua',
'ku': 'kur', 'lo': 'lao', 'la': 'lat', 'lv': 'lav', 'li': 'lim',
'ln': 'lin', 'lt': 'lit', 'lb': 'ltz', 'lu': 'lub', 'lg': 'lug',
'mk': 'mkd', 'mh': 'mah', 'ml': 'mal', 'mi': 'mri', 'mr': 'mar',
'ms': 'msa', 'mg': 'mlg', 'mt': 'mlt', 'mn': 'mon', 'na': 'nau',
'nv': 'nav', 'nr': 'nbl', 'nd': 'nde', 'ng': 'ndo', 'ne': 'nep',
'nn': 'nno', 'nb': 'nob', 'no': 'nor', 'ny': 'nya', 'oc': 'oci',
'oj': 'oji', 'or': 'ori', 'om': 'orm', 'os': 'oss', 'pa': 'pan',
'fa': 'fas', 'pi': 'pli', 'pl': 'pol', 'pt': 'por', 'ps': 'pus',
'qu': 'que', 'rm': 'roh', 'ro': 'ron', 'rn': 'run', 'ru': 'rus',
'sg': 'sag', 'sa': 'san', 'si': 'sin', 'sk': 'slk', 'sl': 'slv',
'se': 'sme', 'sm': 'smo', 'sn': 'sna', 'sd': 'snd', 'so': 'som',
'st': 'sot', 'es': 'spa', 'sc': 'srd', 'sr': 'srp', 'ss': 'ssw',
'su': 'sun', 'sw': 'swa', 'sv': 'swe', 'ty': 'tah', 'ta': 'tam',
'tt': 'tat', 'te': 'tel', 'tg': 'tgk', 'tl': 'tgl', 'th': 'tha',
'bo': 'bod', 'ti': 'tir', 'to': 'ton', 'tn': 'tsn', 'ts': 'tso',
'tk': 'tuk', 'tr': 'tur', 'tw': 'twi', 'ug': 'uig', 'uk': 'ukr',
'ur': 'urd', 'uz': 'uzb', 've': 'ven', 'vi': 'vie', 'vo': 'vol',
'cy': 'cym', 'wa': 'wln', 'wo': 'wol', 'xh': 'xho', 'yi': 'yid',
'yo': 'yor', 'za': 'zha', 'zu': 'zul',
}

const iso639_2to1 = Object.fromEntries(
Object.entries(iso639_1to2).map(([k, v]) => [v, k])
);

const maps = {
2: iso639_1to2,
3: iso639_2to1,
}

export const convertLanguageCode = (code) => {
return maps[code.length]?.[code.toLowerCase()] || null;
}
55 changes: 46 additions & 9 deletions api/src/processing/match-action.js
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,9 @@ import { createResponse } from "./request.js";
import { audioIgnore } from "./service-config.js";
import { createStream } from "../stream/manage.js";
import { splitFilenameExtension } from "../misc/utils.js";
import { convertLanguageCode } from "../misc/language-codes.js";

const extraProcessingTypes = ["merge", "remux", "mute", "audio", "gif"];
const extraProcessingTypes = new Set(["merge", "remux", "mute", "audio", "gif"]);

export default function({
r,
Expand All @@ -19,7 +20,7 @@ export default function({
requestIP,
audioBitrate,
alwaysProxy,
localProcessing
localProcessing,
}) {
let action,
responseType = "tunnel",
Expand All @@ -31,7 +32,10 @@ export default function({
createFilename(r.filenameAttributes, filenameStyle, isAudioOnly, isAudioMuted) : r.filename,
fileMetadata: !disableMetadata ? r.fileMetadata : false,
requestIP,
originalRequest: r.originalRequest
originalRequest: r.originalRequest,
subtitles: r.subtitles,
cover: !disableMetadata ? r.cover : false,
cropCover: !disableMetadata ? r.cropCover : false,
},
params = {};

Expand Down Expand Up @@ -143,7 +147,9 @@ export default function({

case "vimeo":
if (Array.isArray(r.urls)) {
params = { type: "merge" }
params = { type: "merge" };
} else if (r.subtitles) {
params = { type: "remux" };
} else {
responseType = "redirect";
}
Expand All @@ -157,9 +163,22 @@ export default function({
}
break;

case "ok":
case "loom":
if (r.subtitles) {
params = { type: "remux" };
} else {
responseType = "redirect";
}
break;

case "vk":
case "tiktok":
params = {
type: r.subtitles ? "remux" : "proxy"
};
break;

case "ok":
case "xiaohongshu":
params = { type: "proxy" };
break;
Expand All @@ -170,15 +189,14 @@ export default function({
case "pinterest":
case "streamable":
case "snapchat":
case "loom":
case "twitch":
responseType = "redirect";
break;
}
break;

case "audio":
if (audioIgnore.includes(host) || (host === "reddit" && r.typeId === "redirect")) {
if (audioIgnore.has(host) || (host === "reddit" && r.typeId === "redirect")) {
return createResponse("error", {
code: "error.api.service.audio_not_supported"
})
Expand Down Expand Up @@ -226,15 +244,34 @@ export default function({
defaultParams.filename += `.${audioFormat}`;
}

// alwaysProxy is set to true in match.js if localProcessing is forced
if (alwaysProxy && responseType === "redirect") {
responseType = "tunnel";
params.type = "proxy";
}

// TODO: add support for HLS
// (very painful)
if (localProcessing && !params.isHLS && extraProcessingTypes.includes(params.type)) {
responseType = "local-processing";
if (!params.isHLS && responseType !== "picker") {
const isPreferredWithExtra =
localProcessing === "preferred" && extraProcessingTypes.has(params.type);

if (localProcessing === "forced" || isPreferredWithExtra) {
responseType = "local-processing";
}
}

// extractors usually return ISO 639-1 language codes,
// but video players expect ISO 639-2, so we convert them here
if (defaultParams.fileMetadata?.sublanguage?.length === 2) {
const code = convertLanguageCode(defaultParams.fileMetadata.sublanguage);
if (code) {
defaultParams.fileMetadata.sublanguage = code;
} else {
// if a language code couldn't be converted,
// then we don't want it at all
delete defaultParams.fileMetadata.sublanguage;
}
}

return createResponse(
Expand Down
Loading
Loading