be-ytb/js/info.js

parseCodecs = (format) => {
    const mimeType = format['mimeType']
    if (!mimeType) {
        return {};
    }
    const regex = /(?<mimetype>[^/]+\/[^;]+)(?:;\s*codecs="?(?<codecs>[^"]+))?/;
    const match = mimeType.match(regex);
    if (!match) {
        return {};
    }
    const codecs = match.groups.codecs;
    if (!codecs) {
        return {};
    }

    const splitCodecs = codecs.trim().replace(/,$/, '').split(',').map(str => str.trim()).filter(Boolean);
    let vcodec = null;
    let acodec = null;

    for (const fullCodec of splitCodecs) {
        const codec = fullCodec.split('.')[0];

        if (['avc1', 'avc2', 'avc3', 'avc4', 'vp9', 'vp8', 'hev1', 'hev2', 'h263', 'h264', 'mp4v', 'hvc1', 'av01', 'theora'].includes(codec)) {
            if (!vcodec) {
                vcodec = fullCodec;
            }
        } else if (['mp4a', 'opus', 'vorbis', 'mp3', 'aac', 'ac-3', 'ec-3', 'eac3', 'dtsc', 'dtse', 'dtsh', 'dtsl'].includes(codec)) {
            if (!acodec) {
                acodec = fullCodec;
            }
        } else {
            console.log(`WARNING: Unknown codec ${fullCodec}`);
        }
    }

    if (!vcodec && !acodec) {
        if (splitCodecs.length === 2) {
            return {
                vcodec: splitCodecs[0],
                acodec: splitCodecs[1]
            };
        }
    } else {
        return {
            vcodec: vcodec,
            acodec: acodec
        };
    }

    return {};
}

parseSetCookie = (headers) => {
    if (!headers) {
        return ""
    }
    const setCookie = headers['Set-Cookie']
    if (!setCookie) {
        return ""
    }
    console.log(`setCookie: ${setCookie}`)
    let result = 'PREF=hl=en&tz=UTC; SOCS=CAI; GPS=1; ';
    const needCookieNames = ['YSC', 'VISITOR_INFO1_LIVE', 'VISITOR_PRIVACY_METADATA'];
    for (const i in needCookieNames) {
        const cookieName = needCookieNames[i];
        const regexp = new RegExp(`${cookieName}=([^;,]+)`)
        const match = setCookie.match(regexp)
        if (match && match.length === 2) {
            const cookieValue = match[1]
            if (i != needCookieNames.length - 1) {
                result += `${cookieName}=${cookieValue}; `
            } else {
                result += `${cookieName}=${cookieValue}`
            }
        }
    }
    console.log(`current cookie: ${result}`)
    return result;
}

request = async (method, url, data = null, headers = {}, platform) => {
    if (platform === "WEB") {
        url = url.replace("https://www.youtube.com", "http://127.0.0.1");
    }
    console.log(`request url:${url}`)
    console.log(`request data:${data}`)
    console.log(`request method:${method}`)
    console.log(`request headers:${JSON.stringify((headers))}`)
    if (platform === "WEB") {
        const res = await fetch(url, {
            'mode': 'cors',
            'method': method,
            'headers': headers,
            'body': data
        })
        const resData = await res.text()
        return Promise.resolve({
            'data': resData,
            'headers': res.headers
        });
    }
    return new Promise((resolve, reject) => {
        AF.request(url, method, data, headers, (data, headers, err) => {
            if (err) {
                reject(err);
            } else {
                console.log(`response headers: ${headers}`);
                resolve({
                    'data': data,
                    'headers': JSON.parse(headers)
                });
            }
        });
    })
}

getStringBetween = (string, needleStart, needleEnd, offsetStart = 0, offsetEnd = 0) => {
    const x = string.indexOf(needleStart);
    const y = needleEnd ? string.indexOf(needleEnd, x) : string.length;
    return string.substring(x + needleStart.length + offsetEnd, y + offsetStart);
}

findFunction = (jsCode, regexp) => {
    const match = jsCode.match(regexp)
    if (!match && match.length <= 1) {
        return null;
    }
    let result = "";
    const dependencyMatches = match[0].match(/([$a-zA-Z0-9]+\.[$a-zA-Z0-9]+)/g)
    const existDependencies = [];
    if (dependencyMatches && dependencyMatches.length >= 1) {
        for (let currentMatch of dependencyMatches) {
            const varName = currentMatch.split('.')[0];
            if (existDependencies.includes(varName)) {
                continue
            }
            const varNameMatch = jsCode.match(new RegExp(`var \\${varName}={(.|\\n)*?};`), 'ig');
            if (varNameMatch && varNameMatch.length >= 1) {
                result += varNameMatch[0] + "\n";
            }
            existDependencies.push(varName);
        }
    }
    result += `\n${match[0]}`;
    console.log(`decipherFunction result: ` + result);
    return eval(result);
};

const cache = {};
fetchBaseJSContent = async (baseJsUrl, platform) => {
    const cacheKey = `jsContent:${baseJsUrl}`;
    if (cache[cacheKey]) {
        console.log(`from cache: ${baseJsUrl}`);
        return cache[cacheKey];
    }
    console.log(`extract baseUrl: ${baseJsUrl}`);
    const baseContentResp = await request('GET', baseJsUrl, null, {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.101 Safari/537.36',
    }, platform);
    const {data, _} = baseContentResp;
    cache[cacheKey] = data;
    return data;
}

extractJSSignatureFunction = async (baseJsUrl, platform) => {
    const baseJsContent = await fetchBaseJSContent(baseJsUrl, platform);
    return findFunction(baseJsContent, /([a-zA-Z0-9]+)=function\([a-zA-Z0-9]+\)\{a=a\.split\(""\).*};/);
}
extractNJSFunction = async (baseJsUrl, platform) => {
    const baseJsContent = await fetchBaseJSContent(baseJsUrl, platform);
    return findFunction(baseJsContent, /([a-zA-Z0-9]+)=function\([a-zA-Z0-9]+\)\{var b=a\.split\(""\)[\s\S]*?};/);
}

signUrl = async (signatureCipher, baseJsUrl, platform) => {
    const searchParams = {}
    for (const item of signatureCipher.split('&')) {
        const [key, value] = item.split('=');
        searchParams[decodeURIComponent(key)] = decodeURIComponent(value);
    }
    const [url, signature, sp] = [searchParams['url'], searchParams['s'], searchParams['sp']];
    const decipher = await extractJSSignatureFunction(baseJsUrl, platform);
    if (!decipher) {
        return null;
    }
    console.log(`signatureCipher=${signatureCipher}, url=${url}, signature=${signature}, sp=${sp}`)
    let newUrl = `${url}&${sp}=${decipher(signature)}`;

    function replaceUrlParam(url, paramName, paramValue) {
        let pattern = new RegExp(`([?&])${paramName}=.*?(&|$)`, 'i');
        let newUrl = url.replace(pattern, `$1${paramName}=${paramValue}$2`);

        if (newUrl === url && url.indexOf('?') === -1) {
            newUrl += `?${paramName}=${paramValue}`;
        } else if (newUrl === url) {
            newUrl += `&${paramName}=${paramValue}`;
        }
        return newUrl;
    }

    for (const item of url.split('&')) {
        const [key, value] = item.split('=');
        searchParams[decodeURIComponent(key)] = decodeURIComponent(value);
    }

    const nFunction = await extractNJSFunction(baseJsUrl, platform);
    const n = searchParams['n']
    if (n && nFunction) {
        const newN = nFunction(n);
        return replaceUrlParam(newUrl, 'n', newN);
    }
    return newUrl;
}

detail = async (url, platform) => {
    try {
        const htmlResp = await request('GET', `${url}&bpctr=9999999999&has_verified=1`, null, {
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.107 Safari/537.36',
            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
            'Accept-Language': 'en-us,en;q=0.5',
            'Sec-Fetch-Mode': 'navigate',
            'Accept-Encoding': 'gzip, deflate, br',
            'Cookie': 'PREF=hl=en&tz=UTC; SOCS=CAI'
        }, platform);
        let {data: html, headers: htmlHeaders} = htmlResp;

        let regex = /var ytInitialPlayerResponse\s*=\s*({.*?});/;
        let match = html.match(regex);
        if (!match || !match.length) {
            console.log('can not found JSON: ytInitialPlayerResponse');
            throw new Error('JSON not found: ytInitialPlayerResponse');
        }
        const ytInitialPlayerResponse = JSON.parse(match[1]);
        const originVideoDetails = ytInitialPlayerResponse['videoDetails'];
        console.log(`videoDetails: ${JSON.stringify(originVideoDetails)}`);
        const thumbnails = []
        for (const item of originVideoDetails['thumbnail']['thumbnails']) {
            thumbnails.push({
                'url': item['url'],
                'width': item['width'] + "",
                'height': item['height'] + ""
            })
        }

        let originFormats = [];

        const currentFormats = [];
        for (const format of ytInitialPlayerResponse["streamingData"]["formats"].concat(ytInitialPlayerResponse["streamingData"]["adaptiveFormats"])) {
            if (format) {
                format["from"] = "web"
                currentFormats.push(format);
            }
        }
        originFormats = originFormats.concat(currentFormats);
        console.log(`after html, format size:${originFormats.length}`);

        // // android
        // try {
        //     const apiKey = 'AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w'
        //     const apiUrl = `https://www.youtube.com/youtubei/v1/player?key=${apiKey}&prettyPrint=false`;
        //     const apiResp = await request('POST', apiUrl, JSON.stringify({
        //         "context": {
        //             "client": {
        //                 "clientName": "ANDROID",
        //                 "clientVersion": "19.09.37",
        //                 "androidSdkVersion": 30,
        //                 'userAgent': 'com.google.android.youtube/19.09.37 (Linux; U; Android 11) gzip',
        //                 "hl": "en",
        //                 "timeZone": "UTC",
        //                 "utcOffsetMinutes": 0
        //             }
        //         },
        //         'videoId': url.replace('https://www.youtube.com/watch?v=', ''),
        //         "params": "CgIIAQ==",
        //         "playbackContext": {
        //             "contentPlaybackContext": {
        //                 "html5Preference": "HTML5_PREF_WANTS"
        //             }
        //         },
        //         "contentCheckOk": true,
        //         "racyCheckOk": true
        //     }), {
        //         'Host': 'www.youtube.com',
        //         'Connection': 'keep-alive',
        //         'User-Agent': 'com.google.android.youtube/19.09.37 (Linux; U; Android 11) gzip',
        //         'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
        //         'Accept-Language': 'en-us,en;q=0.5',
        //         'Sec-Fetch-Mode': 'navigate',
        //         'X-YouTube-Client-Name': '3',
        //         'X-YouTube-Client-Version': '19.09.37',
        //         'Origin': 'https://www.youtube.com',
        //         'Accept-Encoding': 'gzip, deflate, br',
        //         'Cookie': parseSetCookie(htmlHeaders),
        //         'Content-Type': 'application/json'
        //     }, platform);
        //     let {data: apiData, _} = apiResp;
        //     console.log(`android api result: ${JSON.stringify(apiResp)}`);
        //     const res = JSON.parse(apiData);
        //     const currentFormats = [];
        //     for (const format of [].concat(res["streamingData"]["formats"]).concat(res["streamingData"]["adaptiveFormats"])) {
        //         if (format) {
        //             format["from"] = "android"
        //             currentFormats.push(format);
        //         }
        //     }
        //     originFormats = originFormats.concat(currentFormats);
        // } catch (e) {
        //     console.log(`can not found format android api error: ${e}`);
        // }
        // console.log(`after android api, format size:${originFormats.length}`);
        // // ios
        // try {
        //     const apiKey = 'AIzaSyB-63vPrdThhKuerbB2N_l7Kwwcxj6yUAc'
        //     const apiUrl = `https://www.youtube.com/youtubei/v1/player?key=${apiKey}&prettyPrint=false`;
        //     let apiResp = await request('POST', apiUrl, JSON.stringify({
        //         "context": {
        //             "client": {
        //                 'clientName': 'IOS',
        //                 'clientVersion': '19.09.3',
        //                 'deviceModel': 'iPhone14,3',
        //                 'userAgent': 'com.google.ios.youtube/19.09.3 (iPhone14,3; U; CPU iOS 15_6 like Mac OS X)',
        //                 "hl": "en",
        //                 "timeZone": "UTC",
        //                 "utcOffsetMinutes": 0
        //             }
        //         },
        //         'videoId': url.replace('https://www.youtube.com/watch?v=', ''),
        //         "playbackContext": {
        //             "contentPlaybackContext": {
        //                 "html5Preference": "HTML5_PREF_WANTS"
        //             }
        //         },
        //         "contentCheckOk": true,
        //         "racyCheckOk": true
        //     }), {
        //         'Host': 'www.youtube.com',
        //         'Connection': 'keep-alive',
        //         'User-Agent': 'com.google.ios.youtube/19.09.3 (iPhone14,3; U; CPU iOS 15_6 like Mac OS X)',
        //         'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
        //         'Accept-Language': 'en-us,en;q=0.5',
        //         'Sec-Fetch-Mode': 'navigate',
        //         'X-YouTube-Client-Name': '5',
        //         'X-YouTube-Client-Version': '19.09.3',
        //         'Origin': 'https://www.youtube.com',
        //         'Accept-Encoding': 'gzip, deflate, br',
        //         'Cookie': parseSetCookie(htmlHeaders),
        //         'Content-Type': 'application/json'
        //     }, platform);
        //     let {data: apiData, _} = apiResp;
        //     console.log(`ios api result: ${JSON.stringify(apiResp)}`);
        //     const res = JSON.parse(apiData);
        //     const currentFormats = [];
        //     for (const format of [].concat(res["streamingData"]["formats"]).concat(res["streamingData"]["adaptiveFormats"])) {
        //         if (format) {
        //             format["from"] = "ios"
        //             currentFormats.push(format);
        //         }
        //     }
        //     originFormats = originFormats.concat(currentFormats);
        // } catch (e) {
        //     console.log(`can not found format ios api error: ${e}`);
        // }
        // console.log(`after ios api, format size:${originFormats.length}`);

        const baseJsUrl = `https://www.youtube.com${JSON.parse(html.match(/set\(({.+?})\);/)[1])["PLAYER_JS_URL"]}`
        let formatIds = [];
        const formats = [];
        for (let format of originFormats) {
            console.log(`current format: ${JSON.stringify(format)}`);
            if (format && formatIds.indexOf(format['itag']) === -1) {
                if (!format["url"]) {
                    format["url"] = await signUrl(format["signatureCipher"], baseJsUrl, platform);
                }
                if (format["url"]) {
                    const {vcodec, acodec} = parseCodecs(format)
                    if (vcodec && acodec) {
                        formats.push({
                            "width": format["width"] + "",
                            "height": format["height"] + "",
                            "type": format["mimeType"],
                            "quality": format["quality"],
                            "itag": format["itag"],
                            "fps": format["fps"] + "",
                            "bitrate": format["bitrate"] + "",
                            "url": format["url"],
                            "ext": "mp4",
                            "vcodec": vcodec,
                            "acodec": acodec,
                            "vbr": "0",
                            "abr": "0",
                            "container": "mp4_dash",
                            "from": format["from"]
                        })
                        formatIds.push(format["itag"]);
                    }
                }
            }
        }

        const ytInitialDataMatch = html.match(/var ytInitialData\s*=\s*({.*?});/);
        const recommendInfo = [];
        if (ytInitialDataMatch && ytInitialDataMatch.length === 2) {
            const ytInitialData = JSON.parse(ytInitialDataMatch[1]);
            console.log(`ytInitialData: ${JSON.stringify(ytInitialData)}`);
            for (const item of ytInitialData["contents"]?.["twoColumnWatchNextResults"]?.["secondaryResults"]?.["secondaryResults"]?.["results"] || []) {
                if (item["compactVideoRenderer"]) {
                    const recommendVideo = item["compactVideoRenderer"];
                    console.log(`recommend video: ${JSON.stringify(recommendVideo)}`);
                    if (recommendVideo["videoId"]) {
                        recommendInfo.push({
                            "type": "gridVideoRenderer",
                            "videoId": recommendVideo["videoId"],
                            "title": recommendVideo["title"]?.["simpleText"],
                            "thumbnails": recommendVideo["thumbnail"]?.["thumbnails"],
                            "channelName": recommendVideo["longBylineText"]?.["runs"]?.[0]?.["text"],
                            "publishedTimeText": recommendVideo["publishedTimeText"]?.["simpleText"],
                            "viewCountText": recommendVideo["viewCountText"]?.["simpleText"],
                            "shortViewCountText": recommendVideo["shortViewCountText"]?.["simpleText"],
                            "lengthText": recommendVideo["lengthText"]?.["simpleText"]
                        })
                    }
                }
            }
        }

        const videoDetails = {
            "isLiveContent": originVideoDetails["isLiveContent"],
            "title": originVideoDetails["title"],
            "thumbnails": thumbnails,
            "description": originVideoDetails["shortDescription"],
            "lengthSeconds": originVideoDetails["lengthSeconds"],
            "viewCount": originVideoDetails["viewCount"],
            "keywords": originVideoDetails["keywords"],
            "author": originVideoDetails["author"],
            "channelID": originVideoDetails["channelId"],
            "recommendInfo": recommendInfo,
            "channelURL": `https://www.youtube.com/channel/${originVideoDetails["channelId"]}`,
            "videoId": originVideoDetails["videoId"]
        }
        const ret = {
            "code": 200,
            "msg": "",
            "data": {
                "videoDetails": videoDetails,
                "streamingData": {
                    "formats": formats.reverse()
                }
            },
            "id": "MusicDetailViewModel_detail_url"
        }
        console.log(`detail result: ${JSON.stringify(ret)}`);
        return ret;
    } catch (e) {
        const ret = {
            "code": -1,
            "msg": e.toString()
        }
        console.log(`detail result error: ${JSON.stringify(ret)}`);
        console.log(e);
        return ret;
    }
}

search = async (keyword, next, platform) => {
    try {
        console.log(`search keyword: ${keyword}`);
        console.log(`search next: ${next}`);
        if (next) {
            const nextObject = JSON.parse(next);
            const key = nextObject["key"];
            const body = {
                context: {
                    client: {
                        clientName: "WEB",
                        clientVersion: "2.20240506.01.00",
                    },
                },
                continuation: nextObject["continuation"]
            };
            let res = await request('POST', `https://www.youtube.com/youtubei/v1/search?key=${key}`, JSON.stringify(body), {}, platform);
            const {data, _} = res;
            res = JSON.parse(data);
            const videos = [];
            for (const item of res["onResponseReceivedCommands"][0]["appendContinuationItemsAction"]["continuationItems"][0]["itemSectionRenderer"]["contents"]) {
                const video = item["videoRenderer"];
                console.log(`search result video: ${JSON.stringify(video)}`);
                if (video && video["videoId"]) {
                    videos.push({
                        "type": "videoWithContextRenderer",
                        "data": {
                            "videoId": video["videoId"],
                            "title": video["title"]?.["runs"]?.[0]?.["text"],
                            "thumbnails": video["thumbnail"]?.["thumbnails"],
                            "channelName": video["longBylineText"]?.["runs"]?.[0]?.["text"],
                            "publishedTimeText": video["publishedTimeText"]?.["simpleText"],
                            "viewCountText": video["viewCountText"]?.["simpleText"],
                            "shortViewCountText": video["shortViewCountText"]?.["simpleText"],
                            "lengthText": video["lengthText"]?.["simpleText"]
                        }
                    });
                }
            }
            const ret = {
                "code": 200,
                "msg": "",
                "data": {
                    "data": videos,
                    "next": JSON.stringify({
                        "key": nextObject["key"],
                        "continuation": res["onResponseReceivedCommands"]?.[0]?.["appendContinuationItemsAction"]?.["continuationItems"]?.[1]?.["continuationItemRenderer"]?.["continuationEndpoint"]?.["continuationCommand"]?.["token"],
                    }),
                },
                "id": "MusicSearchResultViewModel_search_result"
            }
            console.log(`[next] search result: ${JSON.stringify(ret)}`);
            return ret;
        } else {
            let url = `https://www.youtube.com/results?q=${encodeURIComponent(keyword)}&sp=EgIQAQ%253D%253D`;

            const htmlRes = await request('GET', url, null, {}, platform);
            const {data: html, _} = htmlRes;

            let regex = /var ytInitialData\s*=\s*({.*?});/;
            let match = html.match(regex);
            if (!match || !match.length) {
                console.log("can not found ytInitialData");
                throw new Error('JSON not found: ytInitialData');
            }

            const ytInitialDataResp = JSON.parse(match[1]);
            const videos = [];
            for (const item of ytInitialDataResp["contents"]?.["twoColumnSearchResultsRenderer"]?.["primaryContents"]?.["sectionListRenderer"]?.["contents"]?.[0]?.["itemSectionRenderer"]?.["contents"]) {
                if (item["videoRenderer"]) {
                    const video = item["videoRenderer"];
                    console.log(`search result video: ${JSON.stringify(video)}`);
                    if (video && video["videoId"]) {
                        videos.push({
                            "type": "videoWithContextRenderer",
                            "data": {
                                "videoId": video["videoId"],
                                "title": video["title"]?.["runs"]?.[0]?.["text"],
                                "thumbnails": video["thumbnail"]?.["thumbnails"],
                                "channelName": video["longBylineText"]?.["runs"]?.[0]?.["text"],
                                "publishedTimeText": video["publishedTimeText"]?.["simpleText"],
                                "viewCountText": video["viewCountText"]?.["simpleText"],
                                "shortViewCountText": video["shortViewCountText"]?.["simpleText"],
                                "lengthText": video["lengthText"]?.["simpleText"]
                            }
                        });
                    }
                }
            }

            let next = {};
            if (html.split("innertubeApiKey").length > 0) {
                next["key"] = html
                    .split("innertubeApiKey")[1]
                    .trim()
                    .split(",")[0]
                    .split('"')[2];
            }
            next["continuation"] = ytInitialDataResp["contents"]?.["twoColumnSearchResultsRenderer"]?.["primaryContents"]?.["sectionListRenderer"]?.["contents"]?.[1]?.["continuationItemRenderer"]?.["continuationEndpoint"]?.["continuationCommand"]?.["token"]

            const ret = {
                "code": 200,
                "msg": "",
                "data": {
                    "data": videos,
                    "next": JSON.stringify(next),
                },
                "id": "MusicSearchResultViewModel_search_result"
            }
            console.log(`unnext search result: ${JSON.stringify(ret)}`);
            return ret;
        }
    } catch (e) {
        const ret = {
            "code": -1,
            "msg": e.toString()
        }
        console.log(`search result error: ${JSON.stringify(ret)}`);
        return ret;
    }
}