be-ytb/js/youtube/youtubev2.js

parseCodecs = (format) => {
    const mimeType = format['mimeType']
    if (!mimeType) {
        return {};
    }
    const regex = /(?<mimetype>[^/]+\/[^;]+)(?:;\s*codecs="?(?<codecs>[^"]+))?/;
    const match = mimeType.match(regex);
    if (!match) {
        return {};
    }
    const codecs = match.groups.codecs;
    if (!codecs) {
        return {};
    }

    const splitCodecs = codecs.trim().replace(/,$/, '').split(',').map(str => str.trim()).filter(Boolean);
    let vcodec = null;
    let acodec = null;

    for (const fullCodec of splitCodecs) {
        const codec = fullCodec.split('.')[0];

        if (['avc1', 'avc2', 'avc3', 'avc4', 'hev1', 'hev2', 'h263', 'h264', 'mp4v', 'hvc1', 'av01', 'theora'].includes(codec)) {
            if (!vcodec) {
                vcodec = fullCodec;
            }
        } else if (['mp4a', 'opus', 'vorbis', 'mp3', 'aac', 'ac-3', 'ec-3', 'eac3', 'dtsc', 'dtse', 'dtsh', 'dtsl'].includes(codec)) {
            if (!acodec) {
                acodec = fullCodec;
            }
        } else {
            console.log(`WARNING: Unknown codec ${fullCodec}`);
        }
    }

    if (!vcodec && !acodec) {
        if (splitCodecs.length === 2) {
            return {
                vcodec: splitCodecs[0], acodec: splitCodecs[1]
            };
        }
    } else {
        return {
            vcodec: vcodec, acodec: acodec
        };
    }

    return {};
}

parseSetCookie = (headers) => {
    if (!headers) {
        return ""
    }
    const setCookie = headers['Set-Cookie']
    if (!setCookie) {
        return ""
    }
    console.log(`setCookie: ${setCookie}`)
    let result = 'PREF=hl=en&tz=UTC; SOCS=CAI; GPS=1; ';
    const needCookieNames = ['YSC', 'VISITOR_INFO1_LIVE', 'VISITOR_PRIVACY_METADATA'];
    for (const i in needCookieNames) {
        const cookieName = needCookieNames[i];
        const regexp = new RegExp(`${cookieName}=([^;,]+)`)
        const match = setCookie.match(regexp)
        if (match && match.length === 2) {
            const cookieValue = match[1]
            if (i !== needCookieNames.length - 1) {
                result += `${cookieName}=${cookieValue}; `
            } else {
                result += `${cookieName}=${cookieValue}`
            }
        }
    }
    console.log(`current cookie: ${result}`)
    return result;
}

request = async (method, url, data = null, headers = {}, requestId, platform) => {
    if (platform === "WEB") {
        url = url.replace("https://www.youtube.com/", "http://16.162.32.168:80/");
        url = url.replace("https://music.youtube.com/", "http://16.162.32.168:80/");
    }
    console.log(`request url:${url}`)
    console.log(`request data:${data}`)
    console.log(`request method:${method}`)
    console.log(`request headers:${JSON.stringify((headers))}`)
    if (platform === "WEB") {
        const res = await fetch(url, {
            'mode': 'cors', 'method': method, 'headers': headers, 'body': data
        })
        const resData = await res.text()
        return Promise.resolve({
            'data': resData, 'headers': res.headers
        });
    }
    return new Promise((resolve, reject) => {
        AF.request(url, method, data, headers, requestId, (data, headers, err) => {
            if (err) {
                reject(err);
            } else {
                console.log(`response headers: ${headers}`);
                resolve({
                    'data': data, 'headers': JSON.parse(headers)
                });
            }
        });
    })
}

detail = async (url, requestId, platform) => {
    try {
        // fetch recommend
        const recommendInfo = [];
        const htmlResp = await request('GET', `${url}&bpctr=9999999999&has_verified=1`, null, {
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.107 Safari/537.36',
            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
            'Accept-Language': 'en-us,en;q=0.5',
            'Sec-Fetch-Mode': 'navigate',
            'Accept-Encoding': 'gzip, deflate, br',
            'Cookie': 'PREF=hl=en&tz=UTC; SOCS=CAI'
        }, requestId, platform);
        let {data: html, headers: htmlHeaders} = htmlResp;
        let regex = /var ytInitialPlayerResponse\s*=\s*({.*?});/;
        let match = html.match(regex);
        if (match) {
            const ytInitialPlayerResponse = JSON.parse(match[1]);
            console.log(ytInitialPlayerResponse);
            const originVideoDetails = ytInitialPlayerResponse['videoDetails'];
            const ytInitialDataMatch = html.match(/var ytInitialData\s*=\s*({.*?});/);
            if (ytInitialDataMatch && ytInitialDataMatch.length === 2) {
                const ytInitialData = JSON.parse(ytInitialDataMatch[1]);
                console.log(ytInitialData);
                for (const item of ytInitialData["contents"]?.["twoColumnWatchNextResults"]?.["secondaryResults"]?.["secondaryResults"]?.["results"] || []) {
                    if (item["compactVideoRenderer"]) {
                        const recommendVideo = item["compactVideoRenderer"];
                        console.log(`recommend video: ${JSON.stringify(recommendVideo)}`);
                        if (recommendVideo["videoId"]) {
                            recommendInfo.push({
                                "type": "gridVideoRenderer",
                                "videoId": recommendVideo["videoId"],
                                "title": recommendVideo["title"]?.["simpleText"],
                                "thumbnails": recommendVideo["thumbnail"]?.["thumbnails"],
                                "channelName": recommendVideo["longBylineText"]?.["runs"]?.[0]?.["text"],
                                "publishedTimeText": recommendVideo["publishedTimeText"]?.["simpleText"],
                                "viewCountText": recommendVideo["viewCountText"]?.["simpleText"],
                                "shortViewCountText": recommendVideo["shortViewCountText"]?.["simpleText"],
                                "lengthText": recommendVideo["lengthText"]?.["simpleText"]
                            })
                        }
                    }
                }
            }
        }

        let thumbnails = [];
        let originFormats = [];
        let originVideoDetails = undefined;
        // android
        try {
            const apiUrl = `https://www.youtube.com/youtubei/v1/player`;
            const apiResp = await request('POST', apiUrl, JSON.stringify({
                "context": {
                    "client": {
                        "clientVersion": "19.50.40",
                        "androidSdkVersion": 30,
                        "clientName": "ANDROID",
                        "osName": "android",
                        "osVersion": "11",
                        "userAgent": "com.google.android.youtube/19.50.40 (Linux; U; Android 11) gzip"
                    }
                },
                "videoId": url.replace('https://www.youtube.com/watch?v=', ''),
                "playbackContext": {
                    "contentPlaybackContext": {
                        "html5Preference": "HTML5_PREF_WANTS"
                    }
                },
                "params": "2AMB"
            }), {
                'Origin': "https://www.youtube.com",
                'X-YouTube-Client-Version': '19.50.40',
                'User-Agent': 'com.google.android.youtube/19.50.40 (Linux; U; Android 11) gzip',
                'Content-Type': 'application/json'
            }, requestId, platform);
            let {data: apiData, _} = apiResp;
            console.log(`android api result: ${JSON.stringify(apiResp)}`);
            const res = JSON.parse(apiData);
            const currentFormats = [];
            originVideoDetails = res["videoDetails"];
            for (const format of [].concat(res["streamingData"]["formats"] || []).concat(res["streamingData"]["adaptiveFormats"] || [])) {
                if (format) {
                    format["from"] = "android"
                    currentFormats.push(format);
                }
            }
            originFormats = originFormats.concat(currentFormats);
        } catch (e) {
            console.log(`can not found format android api error: ${e}`);
            const ret = {
                "code": -1, "msg": e.toString()
            }
            console.log(`detail2 result error: ${JSON.stringify(ret)}`);
            return ret;
        }
        console.log(`after android api, format size:${originFormats.length}`);

        // fallback
        let fallbackFormats = []
        try {
            const apiUrl = `https://www.youtube.com/youtubei/v1/player`;
            const apiResp = await request('POST', apiUrl, JSON.stringify({
                "contentCheckOk": true,
                "context": {
                    "client": {
                        "clientName": "IOS",
                        "clientVersion": "19.47.7",
                        "deviceMake": "Apple",
                        "deviceModel": "iPhone16,2",
                        "hl": "en",
                        "osName": "iPhone",
                        "osVersion": "17.5.1.21F90",
                        "timeZone": "UTC",
                        "userAgent": "com.google.ios.youtube/19.47.7 (iPhone16,2; U; CPU iOS 17_5_1 like Mac OS X;)",
                        "gl": "US",
                        "utcOffsetMinutes": 0
                    }
                },
                "videoId": url.replace('https://www.youtube.com/watch?v=', '')
            }), {
                'User-Agent': 'com.google.ios.youtube/19.47.7 (iPhone16,2; U; CPU iOS 17_5_1 like Mac OS X;)',
                'Content-Type': 'application/json'
            }, requestId, platform);
            let {data: apiData, _} = apiResp;
            console.log(`android api result: ${JSON.stringify(apiResp)}`);
            const res = JSON.parse(apiData);
            const currentFormats = [];
            for (const format of [].concat(res["streamingData"]["formats"] || []).concat(res["streamingData"]["adaptiveFormats"] || [])) {
                if (format) {
                    format["from"] = "android"
                    currentFormats.push(format);
                }
            }
            fallbackFormats = fallbackFormats.concat(currentFormats);
        } catch (e) {
            // console.log(`can not found format android fallback api error: ${e}`);
            // const ret = {
            //     "code": -1, "msg": e.toString()
            // }
            // console.log(`detail2 fallback result error: ${JSON.stringify(ret)}`);
            // return ret;
        }

        let audioUrl = ""
        for (let format of fallbackFormats) {
            if (format["url"]) {
                const {vcodec, acodec} = parseCodecs(format)
                if (!vcodec && acodec) {
                    audioUrl = format["url"]
                    break
                }
            }
        }
        if (audioUrl == "") {
            for (let format in originFormats) {
                if (format["url"]) {
                    const {vcodec, acodec} = parseCodecs(format)
                    if (!vcodec && acodec) {
                        audioUrl = format["url"]
                        break
                    }
                }
            }
        }

        const formats = [];
        const qualities = [];
        for (let format of originFormats) {
            console.log(format);
            if (format["height"] && parseInt(format["height"]) >= 720) {
                continue
            }
            if (format && qualities.indexOf(format['qualityLabel']) === -1) {
                if (format["url"]) {
                    const {vcodec, acodec} = parseCodecs(format)
                    if (vcodec && acodec) {
                        const current = {
                            "width": format["width"] + "",
                            "height": format["height"] + "",
                            "type": format["mimeType"],
                            "quality": format["qualityLabel"],
                            "itag": format["itag"],
                            "fps": format["fps"] + "",
                            "bitrate": format["bitrate"] + "",
                            "ext": "mp4",
                            "vcodec": vcodec,
                            "acodec": acodec,
                            "vbr": "0",
                            "abr": "0",
                            "container": "mp4_dash",
                            "from": format["from"],
                            "url": format["url"],
                            "videoUrl": "",
                            "audioUrl": "",
                            // "videoUrl": format["url"],
                            // "audioUrl": audioUrl
                        }
                        if (platform === "WEB") {
                            current["source"] = format
                        }
                        formats.push(current)
                        qualities.push(format["qualityLabel"]);
                    } else if (vcodec && !acodec) {
                        const current = {
                            "width": format["width"] + "",
                            "height": format["height"] + "",
                            "type": format["mimeType"],
                            "quality": format["qualityLabel"],
                            "itag": format["itag"],
                            "fps": format["fps"] + "",
                            "bitrate": format["bitrate"] + "",
                            "ext": "mp4",
                            "vcodec": vcodec,
                            "acodec": acodec,
                            "vbr": "0",
                            "abr": "0",
                            "container": "mp4_dash",
                            "from": format["from"],
                            "url": "",
                            "videoUrl": format["url"],
                            "audioUrl": audioUrl
                        }
                        if (platform === "WEB") {
                            current["source"] = format
                        }
                        formats.push(current)
                        qualities.push(format["qualityLabel"]);
                    }
                }
            }
        }

        for (const item of originVideoDetails['thumbnail']['thumbnails']) {
            thumbnails.push({
                'url': item['url'], 'width': item['width'] + "", 'height': item['height'] + ""
            })
        }

        formats.sort((a, b) => parseInt(a["height"]) - parseInt(b["height"]));

        const videoDetails = {
            "isLiveContent": originVideoDetails["isLiveContent"],
            "title": originVideoDetails["title"],
            "thumbnails": thumbnails,
            "description": originVideoDetails["shortDescription"],
            "lengthSeconds": originVideoDetails["lengthSeconds"],
            "viewCount": originVideoDetails["viewCount"],
            "keywords": originVideoDetails["keywords"],
            "author": originVideoDetails["author"],
            "channelID": originVideoDetails["channelId"],
            "recommendInfo": recommendInfo,
            "channelURL": `https://www.youtube.com/channel/${originVideoDetails["channelId"]}`,
            "videoId": url.replace('https://www.youtube.com/watch?v=', '')
        }
        const ret = {
            "code": 200, "msg": "", "requestId": requestId, "data": {
                "videoDetails": videoDetails, "streamingData": {
                    "formats": formats
                }
            }, "id": "MusicDetailViewModel_detail_url"
        }
        console.log(`detail result: ${JSON.stringify(ret)}`);
        return ret;
    } catch (e) {
        const ret = {
            "code": -1, "msg": e.toString(), "requestId": requestId,
        }
        console.log(`detail2 result error: ${JSON.stringify(ret)}`);
        console.log(e);
        return ret;
    }
}

search = async (keyword, next, requestId, platform) => {
    try {
        console.log(`search keyword: ${keyword}`);
        console.log(`search next: ${next}`);
        if (next) {
            const nextObject = JSON.parse(next);
            const key = nextObject["key"];
            const body = {
                context: {
                    client: {
                        clientName: "WEB", clientVersion: "2.20240506.01.00",
                    },
                }, continuation: nextObject["continuation"]
            };
            let res = await request('POST', `https://www.youtube.com/youtubei/v1/search?key=${key}`, JSON.stringify(body), {}, requestId, platform);
            const {data, _} = res;
            res = JSON.parse(data);
            const videos = [];
            for (const item of res["onResponseReceivedCommands"][0]["appendContinuationItemsAction"]["continuationItems"][0]["itemSectionRenderer"]["contents"]) {
                const video = item["videoRenderer"];
                console.log(video);
                if (video && video["videoId"] && video["lengthText"]) {
                    videos.push({
                        "type": "videoWithContextRenderer", "data": {
                            "videoId": video["videoId"],
                            "title": video["title"]?.["runs"]?.[0]?.["text"],
                            "thumbnails": video["thumbnail"]?.["thumbnails"],
                            "channelName": video["longBylineText"]?.["runs"]?.[0]?.["text"],
                            "publishedTimeText": video["publishedTimeText"]?.["simpleText"],
                            "viewCountText": video["viewCountText"]?.["simpleText"],
                            "shortViewCountText": video["shortViewCountText"]?.["simpleText"],
                            "lengthText": video["lengthText"]?.["simpleText"]
                        }
                    });
                }
            }
            const ret = {
                "code": 200, "msg": "", "requestId": requestId, "data": {
                    "data": videos, "next": JSON.stringify({
                        "key": nextObject["key"],
                        "continuation": res["onResponseReceivedCommands"]?.[0]?.["appendContinuationItemsAction"]?.["continuationItems"]?.[1]?.["continuationItemRenderer"]?.["continuationEndpoint"]?.["continuationCommand"]?.["token"],
                    }),
                }, "id": "MusicSearchResultViewModel_search_result"
            }
            console.log(`[next] search result: ${JSON.stringify(ret)}`);
            return ret;
        } else {
            let url = `https://www.youtube.com/results?q=${encodeURIComponent(keyword)}&sp=EgIQAQ%253D%253D`;

            const htmlRes = await request('GET', url, null, {}, requestId, platform);
            const {data: html, _} = htmlRes;

            let regex = /var ytInitialData\s*=\s*({.*?});/;
            let match = html.match(regex);
            let json_str = "";
            if (!match || !match.length) {
                let reg = /var ytInitialData = '(.*?)';/;
                match = html.match(reg);
                if (!match) {
                    throw new Error('JSON not found: ytInitialData');
                }
                json_str = match[1].replace(/\\x([0-9a-fA-F]{2})/g, (_, hex) => {
                    return String.fromCharCode(parseInt(hex, 16));
                })
                    .replace(/\\"/g, '"')
                // json_str = match[1].replace(/\\x22/g, '"')
                //     .replace(/\\x5b/g, '[')
                //     .replace(/\\x5d/g, ']')
                //     .replace(/\\x7b/g, '{')
                //     .replace(/\\x7d/g, '}')
                //     .replace(/\\x27/g, "'")
                //     .replace(/\\\x3d/g, "=")
                //     .replace(/\\"/g, '"')
                //     .replace(/\\\//g, '/')
            } else {
                json_str = match[1]
            }

            const ytInitialDataResp = JSON.parse(json_str);
            console.log(ytInitialDataResp);
            const videos = [];
            const contents = ytInitialDataResp["contents"]?.["sectionListRenderer"]?.["contents"] || []
            for (const content of contents) {
                const currentContents = content["itemSectionRenderer"]?.["contents"]
                if (Array.isArray(currentContents)) {
                    for (const currentContent of currentContents) {
                        if (currentContent["videoWithContextRenderer"]) {
                            const video = currentContent["videoWithContextRenderer"];
                            // if (printable(platform)) {
                            //     console.log(video);
                            // }
                            if (video && video["videoId"] && video["lengthText"]) {
                                videos.push({
                                    "type": "videoWithContextRenderer",
                                    "data": {
                                        "videoId": video["videoId"],
                                        "title": video["headline"]?.["runs"]?.[0]?.["text"],
                                        "thumbnails": video["thumbnail"]?.["thumbnails"],
                                        "channelName": video["shortBylineText"]?.["runs"]?.[0]?.["text"],
                                        "publishedTimeText": video["publishedTimeText"]?.["runs"]?.[0]?.["text"],
                                        "viewCountText": video["shortViewCountText"]?.["runs"]?.[0]?.["text"],
                                        "shortViewCountText": video["shortViewCountText"]?.["runs"]?.[0]?.["text"],
                                        "lengthText": video["lengthText"]?.["accessibility"]?.["accessibilityData"]?.["label"],
                                    }
                                });
                            }
                        }
                    }
                }
            }

            let next = {};
            if (html.split("innertubeApiKey").length > 0) {
                next["key"] = html
                    .split("innertubeApiKey")[1]
                    .trim()
                    .split(",")[0]
                    .split('"')[2];
            }
            next["continuation"] = contents[contents.length - 1]?.["continuationItemRenderer"]?.["continuationEndpoint"]?.["continuationCommand"]?.["token"]

            const ret = {
                "code": 200, "msg": "", "requestId": requestId, "data": {
                    "data": videos, "next": JSON.stringify(next),
                }, "id": "MusicSearchResultViewModel_search_result"
            }
            console.log(`unnext search result: ${JSON.stringify(ret)}`);
            return ret;
        }
    } catch (e) {
        const ret = {
            "code": -1, "msg": e.toString(), "requestId": requestId,
        }
        console.log(`search result error: ${JSON.stringify(ret)}`);
        return ret;
    }
}

recommend = async (requestId, platform) => {
    try {
        const body = {
            "context": {
                "client": {
                    "clientName": "WEB", "clientVersion": "2.20240304.00.00"
                }
            }, "browseId": "VLPL4fGSI1pDJn69On1f-8NAvX_CYlx7QyZc"
        }
        let res = await request('POST', `https://www.youtube.com/youtubei/v1/browse?key=AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8`, JSON.stringify(body), {}, requestId, platform);
        const {data, _} = res;
        res = JSON.parse(data);
        const videos = [];
        for (const item of res["contents"]["twoColumnBrowseResultsRenderer"]["tabs"][0]["tabRenderer"]["content"]["sectionListRenderer"]["contents"][0]["itemSectionRenderer"]["contents"]["0"]["playlistVideoListRenderer"]["contents"]) {
            const video = item["playlistVideoRenderer"];
            console.log(video);
            if (video && video["videoId"] && video["lengthText"]) {
                videos.push({
                    "type": "videoWithContextRenderer", "data": {
                        "videoId": video["videoId"],
                        "title": video["title"]?.["runs"]?.[0]?.["text"],
                        "thumbnails": video["thumbnail"]?.["thumbnails"],
                        "channelName": video["longBylineText"]?.["runs"]?.[0]?.["text"],
                        "publishedTimeText": video["publishedTimeText"]?.["simpleText"],
                        "viewCountText": video["viewCountText"]?.["simpleText"],
                        "shortViewCountText": video["shortViewCountText"]?.["simpleText"],
                        "lengthText": video["lengthText"]?.["simpleText"]
                    }
                });
            }
        }
        const ret = {
            "code": 200, "msg": "", "requestId": requestId, "data": {
                "data": videos,
            }, "id": "MusicRecommendResultViewModel_recommend_result"
        }
        console.log(`recommend result: ${JSON.stringify(ret)}`);
        return ret;
    } catch (e) {
        const ret = {
            "code": -1, "msg": e.toString(), "requestId": requestId,
        }
        console.log(`recommend result error: ${JSON.stringify(ret)}`);
        return ret;
    }
}