mirrors/NewPipe
1
0
Fork 0
mirror of https://github.com/TeamNewPipe/NewPipe synced 2025-10-31 15:23:00 +00:00
Code Issues Releases Wiki Activity

Make JavaScript code compatible with older WebViews

Browse Source
This commit is contained in:
Stypox
2025-02-04 21:35:55 +01:00
parent 21df24abfd
commit 53b599b042
4 changed files with 271 additions and 224 deletions
Download Patch File Download Diff File Expand all files Collapse all files

254
app/src/main/assets/po_token.html
View File

@@ -1,204 +1,120 @@
<!DOCTYPE html> <!DOCTYPE html>
<html lang="en"><head><title></title><script> <html lang="en"><head><title></title><script>
class BotGuardClient { /**
constructor(options) { * Factory method to create and load a BotGuardClient instance.
this.userInteractionElement = options.userInteractionElement; * @param options - Configuration options for the BotGuardClient.
this.vm = options.globalObj[options.globalName]; * @returns A promise that resolves to a loaded BotGuardClient instance.
this.program = options.program; */
this.vmFunctions = {}; function loadBotGuard(challengeData) {
this.syncSnapshotFunction = null; this.vm = this[challengeData.globalName];
} this.program = challengeData.program;
this.vmFunctions = {};
this.syncSnapshotFunction = null;
/** if (!this.vm)
* Factory method to create and load a BotGuardClient instance. throw new Error('[BotGuardClient]: VM not found in the global object');
* @param options - Configuration options for the BotGuardClient.
* @returns A promise that resolves to a loaded BotGuardClient instance.
*/
static async create(options) {
return await new BotGuardClient(options).load();
}
async load() { if (!this.vm.a)
if (!this.vm) throw new Error('[BotGuardClient]: Could not load program');
throw new Error('[BotGuardClient]: VM not found in the global object');
if (!this.vm.a) const vmFunctionsCallback = function (
throw new Error('[BotGuardClient]: Could not load program'); asyncSnapshotFunction,
shutdownFunction,
const vmFunctionsCallback = ( passEventFunction,
asyncSnapshotFunction, checkCameraFunction
shutdownFunction, ) {
passEventFunction, this.vmFunctions = {
checkCameraFunction asyncSnapshotFunction: asyncSnapshotFunction,
) => { shutdownFunction: shutdownFunction,
this.vmFunctions = { passEventFunction: passEventFunction,
asyncSnapshotFunction: asyncSnapshotFunction, checkCameraFunction: checkCameraFunction
shutdownFunction: shutdownFunction,
passEventFunction: passEventFunction,
checkCameraFunction: checkCameraFunction
};
}; };
try {
this.syncSnapshotFunction = await this.vm.a(this.program, vmFunctionsCallback, true, this.userInteractionElement, () => {/** no-op */ }, [ [], [] ])[0];
} catch (error) {
throw new Error(`[BotGuardClient]: Failed to load program (${error.message})`);
}
// an asynchronous function runs in the background and it will eventually call
// `vmFunctionsCallback`, however we need to manually tell JavaScript to pass
// control to the things running in the background by interrupting this async
// function in any way, e.g. with a delay of 1ms. The loop is most probably not
// needed but is there just because.
for (let i = 0; i < 10000 && !this.vmFunctions.asyncSnapshotFunction; ++i) {
await new Promise(f => setTimeout(f, 1))
}
return this;
}
/**
* Takes a snapshot asynchronously.
* @returns The snapshot result.
* @example
* ```ts
* const result = await botguard.snapshot({
* contentBinding: {
* c: "a=6&a2=10&b=SZWDwKVIuixOp7Y4euGTgwckbJA&c=1729143849&d=1&t=7200&c1a=1&c6a=1&c6b=1&hh=HrMb5mRWTyxGJphDr0nW2Oxonh0_wl2BDqWuLHyeKLo",
* e: "ENGAGEMENT_TYPE_VIDEO_LIKE",
* encryptedVideoId: "P-vC09ZJcnM"
* }
* });
*
* console.log(result);
* ```
*/
async snapshot(args) {
return new Promise((resolve, reject) => {
if (!this.vmFunctions.asyncSnapshotFunction)
return reject(new Error('[BotGuardClient]: Async snapshot function not found'));
this.vmFunctions.asyncSnapshotFunction((response) => resolve(response), [
args.contentBinding,
args.signedTimestamp,
args.webPoSignalOutput,
args.skipPrivacyBuffer
]);
});
}
}
/**
* Parses the challenge data from the provided response data.
*/
function parseChallengeData(rawData) {
let challengeData = [];
if (rawData.length > 1 && typeof rawData[1] === 'string') {
const descrambled = descramble(rawData[1]);
challengeData = JSON.parse(descrambled || '[]');
} else if (rawData.length && typeof rawData[0] === 'object') {
challengeData = rawData[0];
}
const [ messageId, wrappedScript, wrappedUrl, interpreterHash, program, globalName, , clientExperimentsStateBlob ] = challengeData;
const privateDoNotAccessOrElseSafeScriptWrappedValue = Array.isArray(wrappedScript) ? wrappedScript.find((value) => value && typeof value === 'string') : null;
const privateDoNotAccessOrElseTrustedResourceUrlWrappedValue = Array.isArray(wrappedUrl) ? wrappedUrl.find((value) => value && typeof value === 'string') : null;
return {
messageId,
interpreterJavascript: {
privateDoNotAccessOrElseSafeScriptWrappedValue,
privateDoNotAccessOrElseTrustedResourceUrlWrappedValue
},
interpreterHash,
program,
globalName,
clientExperimentsStateBlob
}; };
this.syncSnapshotFunction = this.vm.a(this.program, vmFunctionsCallback, true, this.userInteractionElement, function () {/** no-op */ }, [ [], [] ])[0]
// an asynchronous function runs in the background and it will eventually call
// `vmFunctionsCallback`, however we need to manually tell JavaScript to pass
// control to the things running in the background by interrupting this async
// function in any way, e.g. with a delay of 1ms. The loop is most probably not
// needed but is there just because.
return new Promise(function (resolve, reject) {
i = 0
refreshIntervalId = setInterval(function () {
if (!!this.vmFunctions.asyncSnapshotFunction) {
resolve(this)
clearInterval(refreshIntervalId);
}
if (i >= 10000) {
reject("asyncSnapshotFunction is null even after 10 seconds")
clearInterval(refreshIntervalId);
}
i += 1;
}, 1);
})
} }
/** /**
* Descrambles the given challenge data. * Takes a snapshot asynchronously.
* @returns The snapshot result.
* @example
* ```ts
* const result = await botguard.snapshot({
* contentBinding: {
* c: "a=6&a2=10&b=SZWDwKVIuixOp7Y4euGTgwckbJA&c=1729143849&d=1&t=7200&c1a=1&c6a=1&c6b=1&hh=HrMb5mRWTyxGJphDr0nW2Oxonh0_wl2BDqWuLHyeKLo",
* e: "ENGAGEMENT_TYPE_VIDEO_LIKE",
* encryptedVideoId: "P-vC09ZJcnM"
* }
* });
*
* console.log(result);
* ```
*/ */
function descramble(scrambledChallenge) { function snapshot(args) {
const buffer = base64ToU8(scrambledChallenge); return new Promise(function (resolve, reject) {
if (buffer.length) if (!this.vmFunctions.asyncSnapshotFunction)
return new TextDecoder().decode(buffer.map((b) => b + 97)); return reject(new Error('[BotGuardClient]: Async snapshot function not found'));
this.vmFunctions.asyncSnapshotFunction(function (response) { resolve(response) }, [
args.contentBinding,
args.signedTimestamp,
args.webPoSignalOutput,
args.skipPrivacyBuffer
]);
});
} }
const base64urlCharRegex = /[-_.]/g; function runBotGuard(challengeData) {
const base64urlToBase64Map = {
'-': '+',
_: '/',
'.': '='
};
function base64ToU8(base64) {
let base64Mod;
if (base64urlCharRegex.test(base64)) {
base64Mod = base64.replace(base64urlCharRegex, function (match) {
return base64urlToBase64Map[match];
});
} else {
base64Mod = base64;
}
base64Mod = atob(base64Mod);
return new Uint8Array(
[ ...base64Mod ].map(
(char) => char.charCodeAt(0)
)
);
}
function u8ToBase64(u8, base64url = false) {
const result = btoa(String.fromCharCode(...u8));
if (base64url) {
return result
.replace(/\+/g, '-')
.replace(/\//g, '_');
}
return result;
}
async function runBotGuard(rawChallengeData) {
const challengeData = parseChallengeData(rawChallengeData)
const interpreterJavascript = challengeData.interpreterJavascript.privateDoNotAccessOrElseSafeScriptWrappedValue; const interpreterJavascript = challengeData.interpreterJavascript.privateDoNotAccessOrElseSafeScriptWrappedValue;
if (interpreterJavascript) { if (interpreterJavascript) {
new Function(interpreterJavascript)(); new Function(interpreterJavascript)();
} else throw new Error('Could not load VM'); } else throw new Error('Could not load VM');
const botguard = await BotGuardClient.create({ const webPoSignalOutput = [];
return loadBotGuard({
globalName: challengeData.globalName, globalName: challengeData.globalName,
globalObj: this, globalObj: this,
program: challengeData.program program: challengeData.program
}); }).then(function (botguard) {
return botguard.snapshot({ webPoSignalOutput: webPoSignalOutput })
const webPoSignalOutput = []; }).then(function (botguardResponse) {
const botguardResponse = await botguard.snapshot({ webPoSignalOutput }); return { webPoSignalOutput: webPoSignalOutput, botguardResponse: botguardResponse }
return { webPoSignalOutput, botguardResponse } })
} }
async function obtainPoToken(webPoSignalOutput, integrityTokenResponse, identifier) { function obtainPoToken(webPoSignalOutput, integrityToken, identifier) {
const integrityToken = integrityTokenResponse[0];
const getMinter = webPoSignalOutput[0]; const getMinter = webPoSignalOutput[0];
if (!getMinter) if (!getMinter)
throw new Error('PMD:Undefined'); throw new Error('PMD:Undefined');
const mintCallback = await getMinter(base64ToU8(integrityToken)); const mintCallback = getMinter(integrityToken);
if (!(mintCallback instanceof Function)) if (!(mintCallback instanceof Function))
throw new Error('APF:Failed'); throw new Error('APF:Failed');
const result = await mintCallback(new TextEncoder().encode(identifier)); const result = mintCallback(identifier);
if (!result) if (!result)
throw new Error('YNJ:Undefined'); throw new Error('YNJ:Undefined');
@@ -206,6 +122,6 @@ async function obtainPoToken(webPoSignalOutput, integrityTokenResponse, identifi
if (!(result instanceof Uint8Array)) if (!(result instanceof Uint8Array))
throw new Error('ODM:Invalid'); throw new Error('ODM:Invalid');
return u8ToBase64(result, true); return result;
} }
</script></head><body></body></html> </script></head><body></body></html>

113
app/src/main/java/org/schabi/newpipe/util/potoken/JavaScriptUtil.kt Normal file
View File

@@ -0,0 +1,113 @@
package org.schabi.newpipe.util.potoken
import com.grack.nanojson.JsonObject
import com.grack.nanojson.JsonParser
import com.grack.nanojson.JsonWriter
import okio.ByteString.Companion.decodeBase64
import okio.ByteString.Companion.toByteString
/**
* Parses the raw challenge data obtained from the Create endpoint and returns an object that can be
* embedded in a JavaScript snippet.
*/
fun parseChallengeData(rawChallengeData: String): String {
val scrambled = JsonParser.array().from(rawChallengeData)
val challengeData = if (scrambled.size > 1 && scrambled.isString(1)) {
val descrambled = descramble(scrambled.getString(1))
JsonParser.array().from(descrambled)
} else {
scrambled.getArray(1)
}
val messageId = challengeData.getString(0)
val interpreterHash = challengeData.getString(3)
val program = challengeData.getString(4)
val globalName = challengeData.getString(5)
val clientExperimentsStateBlob = challengeData.getString(7)
val privateDoNotAccessOrElseSafeScriptWrappedValue = challengeData.getArray(1, null)?.find { it is String }
val privateDoNotAccessOrElseTrustedResourceUrlWrappedValue = challengeData.getArray(2, null)?.find { it is String }
return JsonWriter.string(
JsonObject.builder()
.value("messageId", messageId)
.`object`("interpreterJavascript")
.value("privateDoNotAccessOrElseSafeScriptWrappedValue", privateDoNotAccessOrElseSafeScriptWrappedValue)
.value("privateDoNotAccessOrElseTrustedResourceUrlWrappedValue", privateDoNotAccessOrElseTrustedResourceUrlWrappedValue)
.end()
.value("interpreterHash", interpreterHash)
.value("program", program)
.value("globalName", globalName)
.value("clientExperimentsStateBlob", clientExperimentsStateBlob)
.done()
)
}
/**
* Parses the raw integrity token data obtained from the GenerateIT endpoint to a JavaScript
* `Uint8Array` that can be embedded directly in JavaScript code, and an [Int] representing the
* duration of this token in seconds.
*/
fun parseIntegrityTokenData(rawIntegrityTokenData: String): Pair<String, Long> {
val integrityTokenData = JsonParser.array().from(rawIntegrityTokenData)
return base64ToU8(integrityTokenData.getString(0)) to integrityTokenData.getLong(1)
}
/**
* Converts a string (usually the identifier used as input to `obtainPoToken`) to a JavaScript
* `Uint8Array` that can be embedded directly in JavaScript code.
*/
fun stringToU8(identifier: String): String {
return newUint8Array(identifier.toByteArray())
}
/**
* Takes a poToken encoded as a sequence of bytes represented as integers separated by commas
* (e.g. "97,98,99" would be "abc"), which is the output of `Uint8Array::toString()` in JavaScript,
* and converts it to the specific base64 representation for poTokens.
*/
fun u8ToBase64(poToken: String): String {
return poToken.split(",")
.map { it.toUByte().toByte() }
.toByteArray()
.toByteString()
.base64()
.replace("+", "-")
.replace("/", "_")
}
/**
* Takes the scrambled challenge, decodes it from base64, adds 97 to each byte.
*/
private fun descramble(scrambledChallenge: String): String {
return base64ToByteString(scrambledChallenge)
.map { (it + 97).toByte() }
.toByteArray()
.decodeToString()
}
/**
* Decodes a base64 string encoded in the specific base64 representation used by YouTube, and
* returns a JavaScript `Uint8Array` that can be embedded directly in JavaScript code.
*/
private fun base64ToU8(base64: String): String {
return newUint8Array(base64ToByteString(base64))
}
private fun newUint8Array(contents: ByteArray): String {
return "new Uint8Array([" + contents.joinToString(separator = ",") { it.toUByte().toString() } + "])"
}
/**
* Decodes a base64 string encoded in the specific base64 representation used by YouTube.
*/
private fun base64ToByteString(base64: String): ByteArray {
val base64Mod = base64
.replace('-', '+')
.replace('_', '/')
.replace('.', '=')
return (base64Mod.decodeBase64() ?: throw PoTokenException("Cannot base64 decode"))
.toByteArray()
}

12
app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenProviderImpl.kt
View File

@@ -58,12 +58,6 @@ object PoTokenProviderImpl : PoTokenProvider {
webPoTokenGenerator!!.isExpired() webPoTokenGenerator!!.isExpired()
if (shouldRecreate) { if (shouldRecreate) {
// close the current webPoTokenGenerator on the main thread
webPoTokenGenerator?.let { Handler(Looper.getMainLooper()).post { it.close() } }
// create a new webPoTokenGenerator
webPoTokenGenerator = PoTokenWebView
.newPoTokenGenerator(App.getApp()).blockingGet()
val innertubeClientRequestInfo = InnertubeClientRequestInfo.ofWebClient() val innertubeClientRequestInfo = InnertubeClientRequestInfo.ofWebClient()
innertubeClientRequestInfo.clientInfo.clientVersion = innertubeClientRequestInfo.clientInfo.clientVersion =
@@ -78,6 +72,12 @@ object PoTokenProviderImpl : PoTokenProvider {
null, null,
false false
) )
// close the current webPoTokenGenerator on the main thread
webPoTokenGenerator?.let { Handler(Looper.getMainLooper()).post { it.close() } }
// create a new webPoTokenGenerator
webPoTokenGenerator = PoTokenWebView
.newPoTokenGenerator(App.getApp()).blockingGet()
// The streaming poToken needs to be generated exactly once before generating // The streaming poToken needs to be generated exactly once before generating
// any other (player) tokens. // any other (player) tokens.

116
app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenWebView.kt
View File

@@ -52,10 +52,11 @@ class PoTokenWebView private constructor(
// supports a really old version of JS. // supports a really old version of JS.
val fmt = "\"${m.message()}\", source: ${m.sourceId()} (${m.lineNumber()})" val fmt = "\"${m.message()}\", source: ${m.sourceId()} (${m.lineNumber()})"
val exception = BadWebViewException(fmt)
Log.e(TAG, "This WebView implementation is broken: $fmt") Log.e(TAG, "This WebView implementation is broken: $fmt")
// This can only happen during initialization, where there is no try-catch onInitializationErrorCloseAndCancel(exception)
onInitializationErrorCloseAndCancel(BadWebViewException(fmt)) popAllPoTokenEmitters().forEach { (_, emitter) -> emitter.onError(exception) }
} }
return super.onConsoleMessage(m) return super.onConsoleMessage(m)
} }
@@ -84,7 +85,7 @@ class PoTokenWebView private constructor(
{ html -> { html ->
webView.loadDataWithBaseURL( webView.loadDataWithBaseURL(
"https://www.youtube.com", "https://www.youtube.com",
html.replace( html.replaceFirst(
"</script>", "</script>",
// calls downloadAndRunBotguard() when the page has finished loading // calls downloadAndRunBotguard() when the page has finished loading
"\n$JS_INTERFACE.downloadAndRunBotguard()</script>" "\n$JS_INTERFACE.downloadAndRunBotguard()</script>"
@@ -113,18 +114,21 @@ class PoTokenWebView private constructor(
"https://www.youtube.com/api/jnn/v1/Create", "https://www.youtube.com/api/jnn/v1/Create",
"[ \"$REQUEST_KEY\" ]", "[ \"$REQUEST_KEY\" ]",
) { responseBody -> ) { responseBody ->
val parsedChallengeData = parseChallengeData(responseBody)
webView.evaluateJavascript( webView.evaluateJavascript(
"""(async function() { """try {
try { data = $parsedChallengeData
data = JSON.parse(String.raw`$responseBody`) runBotGuard(data).then(function (result) {
result = await runBotGuard(data)
this.webPoSignalOutput = result.webPoSignalOutput this.webPoSignalOutput = result.webPoSignalOutput
$JS_INTERFACE.onRunBotguardResult(result.botguardResponse) $JS_INTERFACE.onRunBotguardResult(result.botguardResponse)
} catch (error) { }, function (error) {
$JS_INTERFACE.onJsInitializationError(error.toString()) $JS_INTERFACE.onJsInitializationError(error + "\n" + error.stack)
} })
})();""", } catch (error) {
) {} $JS_INTERFACE.onJsInitializationError(error + "\n" + error.stack)
}""",
null
)
} }
} }
@@ -156,38 +160,24 @@ class PoTokenWebView private constructor(
if (BuildConfig.DEBUG) { if (BuildConfig.DEBUG) {
Log.d(TAG, "GenerateIT response: $responseBody") Log.d(TAG, "GenerateIT response: $responseBody")
} }
webView.evaluateJavascript( val (integrityToken, expirationTimeInSeconds) = parseIntegrityTokenData(responseBody)
"""(async function() {
try {
this.integrityToken = JSON.parse(String.raw`$responseBody`)
$JS_INTERFACE.onInitializationFinished(integrityToken[1])
} catch (error) {
$JS_INTERFACE.onJsInitializationError(error.toString())
}
})();""",
) {}
}
}
/** // leave 10 minutes of margin just to be sure
* Called during initialization by the JavaScript snippet from [onRunBotguardResult] when the expirationInstant = Instant.now().plusSeconds(expirationTimeInSeconds - 600)
* `integrityToken` has been received by JavaScript.
* webView.evaluateJavascript(
* @param expirationTimeInSeconds in how many seconds the integrity token expires, can be found "this.integrityToken = $integrityToken"
* in `integrityToken[1]` ) {
*/ if (BuildConfig.DEBUG) {
@JavascriptInterface Log.d(TAG, "initialization finished, expiration=${expirationTimeInSeconds}s")
fun onInitializationFinished(expirationTimeInSeconds: Long) { }
if (BuildConfig.DEBUG) { generatorEmitter.onSuccess(this)
Log.d(TAG, "onInitializationFinished() called, expiration=${expirationTimeInSeconds}s") }
} }
// leave 10 minutes of margin just to be sure
expirationInstant = Instant.now().plusSeconds(expirationTimeInSeconds - 600)
generatorEmitter.onSuccess(this)
} }
//endregion //endregion
//region Obtaining poTokens //region Handling multiple emitters
/** /**
* Adds the ([identifier], [emitter]) pair to the [poTokenEmitters] list. This makes it so that * Adds the ([identifier], [emitter]) pair to the [poTokenEmitters] list. This makes it so that
* multiple poToken requests can be generated invparallel, and the results will be notified to * multiple poToken requests can be generated invparallel, and the results will be notified to
@@ -212,6 +202,20 @@ class PoTokenWebView private constructor(
} }
} }
/**
* Clears [poTokenEmitters] and returns its previous contents. The emitters are supposed to be
* used immediately after to either signal a success or an error.
*/
private fun popAllPoTokenEmitters(): List<Pair<String, SingleEmitter<String>>> {
return synchronized(poTokenEmitters) {
val result = poTokenEmitters.toList()
poTokenEmitters.clear()
result
}
}
//endregion
//region Obtaining poTokens
override fun generatePoToken(identifier: String): Single<String> = override fun generatePoToken(identifier: String): Single<String> =
Single.create { emitter -> Single.create { emitter ->
if (BuildConfig.DEBUG) { if (BuildConfig.DEBUG) {
@@ -219,17 +223,21 @@ class PoTokenWebView private constructor(
} }
runOnMainThread(emitter) { runOnMainThread(emitter) {
addPoTokenEmitter(identifier, emitter) addPoTokenEmitter(identifier, emitter)
val u8Identifier = stringToU8(identifier)
webView.evaluateJavascript( webView.evaluateJavascript(
"""(async function() { """try {
identifier = String.raw`$identifier` identifier = "$identifier"
try { u8Identifier = $u8Identifier
poToken = await obtainPoToken(webPoSignalOutput, integrityToken, poTokenU8 = obtainPoToken(webPoSignalOutput, integrityToken, u8Identifier)
identifier) poTokenU8String = ""
$JS_INTERFACE.onObtainPoTokenResult(identifier, poToken) for (i = 0; i < poTokenU8.length; i++) {
} catch (error) { if (i != 0) poTokenU8String += ","
$JS_INTERFACE.onObtainPoTokenError(identifier, error.toString()) poTokenU8String += poTokenU8[i]
} }
})();""", $JS_INTERFACE.onObtainPoTokenResult(identifier, poTokenU8String)
} catch (error) {
$JS_INTERFACE.onObtainPoTokenError(identifier, error + "\n" + error.stack)
}""",
) {} ) {}
} }
} }
@@ -251,7 +259,17 @@ class PoTokenWebView private constructor(
* result of the JavaScript `obtainPoToken()` function. * result of the JavaScript `obtainPoToken()` function.
*/ */
@JavascriptInterface @JavascriptInterface
fun onObtainPoTokenResult(identifier: String, poToken: String) { fun onObtainPoTokenResult(identifier: String, poTokenU8: String) {
if (BuildConfig.DEBUG) {
Log.d(TAG, "Generated poToken (before decoding): identifier=$identifier poTokenU8=$poTokenU8")
}
val poToken = try {
u8ToBase64(poTokenU8)
} catch (t: Throwable) {
popPoTokenEmitter(identifier)?.onError(t)
return
}
if (BuildConfig.DEBUG) { if (BuildConfig.DEBUG) {
Log.d(TAG, "Generated poToken: identifier=$identifier poToken=$poToken") Log.d(TAG, "Generated poToken: identifier=$identifier poToken=$poToken")
} }