爬蟲技術探索：批次將Youtube影片加入播放清單

前言

上次介紹了簡單應用 Javascript，抓出想要的資料進行處理的方法，但是其實瀏覽器的開發者工具還有很多方便的功能，這次要做的是稍微進階一點的爬蟲應用

操作方式

開啟瀏覽器的開發者工具 (F12 或右鍵檢查)，切換到 Network (網路) 分頁
把影片加入播放清單，並觀察開發者工具的變化
找出剛剛加入播放清單後增加的幾筆紀錄，點擊查看詳細內容
在 Header、Payload 分頁可以看到傳送的各種參數，這些就是加入播放清單會送給伺服器的資料
測試一下，右鍵 -> Copy -> Copy as fetch，fetch 是瀏覽器用來與伺服器溝通的一種方式，有興趣進一步了解可以看看 MDN
把剛剛複製的內容，貼在下面控制台 (console) 並執行 (Enter)
回去檢查播放清單，應該會發現你剛剛選擇的影片加入了兩次，由此確定找到了需要的資料

回到開發者工具，通常 Header 是用來做一些驗證，而 Payload 是一些要溝通的資料，這裡發現有一筆資料

"ACTION_ADD_VIDEO", addedVideoId: "..."}```，下面有一段```playlistId: "..."```，從英文就可以看出，這是決定哪部影片，加入哪個播放清單的地方，因此在執行 fetch 的時候修改這些地方就可以了

9. 為了方便使用，可以把它用 Javascript 包裝一下，這裡結合了上一篇提到的一些應用，簡單的範例如下：
```var LibbyYoutubeFunc = {
    key: 'YOUR_KEY',
    playlist: 'YOUR_PLAYLIST_ID',
    authorization: 'YOUR_AUTHORIZATION',
    continuation: '',
    videos: [],
    // 取得影片ID getVideos(數量, 從第幾部影片開始)
    getVideos: (cnt = 30, offset = 0) => Array.from(document.querySelectorAll('a#video-title')).map(v=> v.href.replace(/.+(?:\?v=|shorts\/)([^&]+)/, '$1').substring(0,11)).slice(offset, offset+cnt),
    // 批次將影片加入播放清單 addVideos(數量, 從第幾部影片開始)
    addVideos(cnt = 30, offset = 0){
        this.addVideoToList(this.getVideos(cnt, offset));
    },
    // 取得畫面上的影片數量 countVideos()
    countVideos: () => Array.from(document.querySelectorAll('a#video-title')).length,
    // 取得畫面上的影片連結 getLinks(數量, 從第幾部影片開始)
    getLinks: (cnt = 30, offset = 0) => Array.from(document.querySelectorAll('a#video-title')).slice(offset, offset+cnt).map(v=>v.href),
    // 將影片加入播放清單 addVideoToList(影片ID)
    addVideoToList(videos){
        if(!videos) return;
        let actions = JSON.stringify(videos.map(v=>{ return {'action':'ACTION_ADD_VIDEO','addedVideoId':v} }));
        fetch(`https://www.youtube.com/youtubei/v1/browse/edit_playlist?key=${this.key}&prettyPrint=false`, {
            "headers": {
              "accept": "*/*",
              "accept-language": "zh-TW,zh;q=0.9,en-US;q=0.8,en;q=0.7",
              "authorization": this.authorization,
              "content-type": "application/json",
              "sec-ch-ua": "\"Chromium\";v=\"96\", \" Not A;Brand\";v=\"99\"",
              "sec-ch-ua-arch": "\"x86\"",
              "sec-ch-ua-bitness": "\"64\"",
              "sec-ch-ua-full-version": "\"96.0.4664.137\"",
              "sec-ch-ua-mobile": "?0",
              "sec-ch-ua-model": "",
              "sec-ch-ua-platform": "\"Windows\"",
              "sec-ch-ua-platform-version": "\"10.0.0\"",
              "sec-fetch-dest": "empty",
              "sec-fetch-mode": "same-origin",
              "sec-fetch-site": "same-origin",
              "x-goog-authuser": "0",
              "x-goog-pageid": "112278217211704261332",
              "x-goog-visitor-id": "Cgt5T09vSWw1ZEtzVSiNhaWXBg%3D%3D",
              "x-origin": "https://www.youtube.com",
              "x-youtube-bootstrap-logged-in": "true",
              "x-youtube-client-name": "1",
              "x-youtube-client-version": "2.20220801.00.00"
            },
            "referrer": location.href,
            "referrerPolicy": "strict-origin-when-cross-origin",
            "body": `{\"context\":{\"client\":{\"hl\":\"ru\",\"gl\":\"TW\",\"remoteHost\":\"114.24.219.231\",\"deviceMake\":\"\",\"deviceModel\":\"\",\"visitorData\":\"Cgt5T09vSWw1ZEtzVSiNhaWXBg%3D%3D\",\"userAgent\":\"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.137 Safari/537.36,gzip(gfe)\",\"clientName\":\"WEB\",\"clientVersion\":\"2.20220801.00.00\",\"osName\":\"Windows\",\"osVersion\":\"10.0\",\"originalUrl\":\"https://www.youtube.com/channel/UCxWAL-c1psONO_DfG-cq2iA\",\"platform\":\"DESKTOP\",\"clientFormFactor\":\"UNKNOWN_FORM_FACTOR\",\"configInfo\":{\"appInstallData\":\"CI2FpZcGEMvs_RIQt8utBRDLoq4FEKbz_RIQ1IOuBRDm-P0SEMT2_RIQuIuuBRDYmq4FEJa1rgUQ0K2uBRD9-v0SENi-rQUQkfj8Eg%3D%3D\"},\"timeZone\":\"Asia/Taipei\",\"browserName\":\"Chrome\",\"browserVersion\":\"96.0.4664.137\",\"screenWidthPoints\":1305,\"screenHeightPoints\":954,\"screenPixelDensity\":1,\"screenDensityFloat\":1,\"utcOffsetMinutes\":480,\"userInterfaceTheme\":\"USER_INTERFACE_THEME_LIGHT\",\"connectionType\":\"CONN_CELLULAR_4G\",\"memoryTotalKbytes\":\"8000000\",\"mainAppWebInfo\":{\"graftUrl\":\"https://www.youtube.com/channel/UCxWAL-c1psONO_DfG-cq2iA/videos\",\"pwaInstallabilityStatus\":\"PWA_INSTALLABILITY_STATUS_CAN_BE_INSTALLED\",\"webDisplayMode\":\"WEB_DISPLAY_MODE_BROWSER\",\"isWebNativeShareAvailable\":true}},\"user\":{\"lockedSafetyMode\":false},\"request\":{\"useSsl\":true,\"internalExperimentFlags\":[],\"consistencyTokenJars\":[]},\"clickTracking\":{\"clickTrackingParams\":\"CAAQisQGIhMIkYjY6ryo-QIViB5gCh1NawNv\"},\"adSignalsInfo\":{\"params\":[{\"key\":\"dt\",\"value\":\"1659454091942\"},{\"key\":\"flash\",\"value\":\"0\"},{\"key\":\"frm\",\"value\":\"0\"},{\"key\":\"u_tz\",\"value\":\"480\"},{\"key\":\"u_his\",\"value\":\"2\"},{\"key\":\"u_h\",\"value\":\"1080\"},{\"key\":\"u_w\",\"value\":\"1920\"},{\"key\":\"u_ah\",\"value\":\"1040\"},{\"key\":\"u_aw\",\"value\":\"1920\"},{\"key\":\"u_cd\",\"value\":\"24\"},{\"key\":\"bc\",\"value\":\"31\"},{\"key\":\"bih\",\"value\":\"954\"},{\"key\":\"biw\",\"value\":\"1288\"},{\"key\":\"brdim\",\"value\":\"0,0,0,0,1920,0,1920,1040,1305,954\"},{\"key\":\"vis\",\"value\":\"1\"},{\"key\":\"wgl\",\"value\":\"true\"},{\"key\":\"ca_type\",\"value\":\"image\"}]}},\"actions\":${actions},"playlistId":"${this.playlist}"}`,
            "method": "POST",
            "mode": "cors",
            "credentials": "include"
        }).then(res=>res.json()).then(data=>{
            if(data.error) console.log(data.error, videos);
        });
    },
    // 模擬捲動行為，取得後面的影片 (未充分測試)
    getNextVideos(){
        if(!this.continuation) return;
        fetch(`https://www.youtube.com/youtubei/v1/browse?key=${this.key}&prettyPrint=false`, {
        "headers": {
            "accept": "*/*",
            "accept-language": "zh-TW,zh;q=0.9,en-US;q=0.8,en;q=0.7",
            "authorization": this.authorization,
            "content-type": "application/json",
            "sec-ch-ua": "\"Chromium\";v=\"96\", \" Not A;Brand\";v=\"99\"",
            "sec-ch-ua-arch": "\"x86\"",
            "sec-ch-ua-bitness": "\"64\"",
            "sec-ch-ua-full-version": "\"96.0.4664.137\"",
            "sec-ch-ua-mobile": "?0",
            "sec-ch-ua-model": "",
            "sec-ch-ua-platform": "\"Windows\"",
            "sec-ch-ua-platform-version": "\"10.0.0\"",
            "sec-fetch-dest": "empty",
            "sec-fetch-mode": "same-origin",
            "sec-fetch-site": "same-origin",
            "x-goog-authuser": "0",
            "x-goog-pageid": "112278217211704261332",
            "x-goog-visitor-id": "Cgt5T09vSWw1ZEtzVSimuaWXBg%3D%3D",
            "x-origin": "https://www.youtube.com",
            "x-youtube-bootstrap-logged-in": "true",
            "x-youtube-client-name": "1",
            "x-youtube-client-version": "2.20220801.00.00"
        },
        "referrer": "https://www.youtube.com/channel/UC_soGpcb7SORsk5yzHLMdTQ/videos",
        "referrerPolicy": "origin-when-cross-origin",
        "body": `{\"context\":{\"client\":{\"hl\":\"ru\",\"gl\":\"TW\",\"remoteHost\":\"114.24.219.231\",\"deviceMake\":\"\",\"deviceModel\":\"\",\"visitorData\":\"Cgt5T09vSWw1ZEtzVSimuaWXBg%3D%3D\",\"userAgent\":\"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.137 Safari/537.36,gzip(gfe)\",\"clientName\":\"WEB\",\"clientVersion\":\"2.20220801.00.00\",\"osName\":\"Windows\",\"osVersion\":\"10.0\",\"originalUrl\":\"https://www.youtube.com/watch?v=1rIlWGVD8u8\",\"platform\":\"DESKTOP\",\"clientFormFactor\":\"UNKNOWN_FORM_FACTOR\",\"configInfo\":{\"appInstallData\":\"CKa5pZcGENiargUQ0K2uBRC3y60FELiLrgUQ5vj9EhDUg64FEKbz_RIQy6KuBRDL7P0SEMT2_RIQlrWuBRD9-v0SENi-rQUQkfj8Eg%3D%3D\"},\"timeZone\":\"Asia/Taipei\",\"browserName\":\"Chrome\",\"browserVersion\":\"96.0.4664.137\",\"screenWidthPoints\":1920,\"screenHeightPoints\":954,\"screenPixelDensity\":1,\"screenDensityFloat\":1,\"utcOffsetMinutes\":480,\"userInterfaceTheme\":\"USER_INTERFACE_THEME_LIGHT\",\"connectionType\":\"CONN_CELLULAR_4G\",\"memoryTotalKbytes\":\"8000000\",\"mainAppWebInfo\":{\"graftUrl\":\"https://www.youtube.com/channel/UC_soGpcb7SORsk5yzHLMdTQ/videos\",\"pwaInstallabilityStatus\":\"PWA_INSTALLABILITY_STATUS_CAN_BE_INSTALLED\",\"webDisplayMode\":\"WEB_DISPLAY_MODE_BROWSER\",\"isWebNativeShareAvailable\":true}},\"user\":{\"lockedSafetyMode\":false},\"request\":{\"useSsl\":true,\"internalExperimentFlags\":[],\"consistencyTokenJars\":[]},\"clickTracking\":{\"clickTrackingParams\":\"CC8Q6IsCGAAiEwjdnaeU1aj5AhUDxUwCHZeIB8s=\"},\"adSignalsInfo\":{\"params\":[{\"key\":\"dt\",\"value\":\"1659460773110\"},{\"key\":\"flash\",\"value\":\"0\"},{\"key\":\"frm\",\"value\":\"0\"},{\"key\":\"u_tz\",\"value\":\"480\"},{\"key\":\"u_his\",\"value\":\"3\"},{\"key\":\"u_h\",\"value\":\"1080\"},{\"key\":\"u_w\",\"value\":\"1920\"},{\"key\":\"u_ah\",\"value\":\"1040\"},{\"key\":\"u_aw\",\"value\":\"1920\"},{\"key\":\"u_cd\",\"value\":\"24\"},{\"key\":\"bc\",\"value\":\"31\"},{\"key\":\"bih\",\"value\":\"954\"},{\"key\":\"biw\",\"value\":\"1903\"},{\"key\":\"brdim\",\"value\":\"0,0,0,0,1920,0,1920,1040,1920,954\"},{\"key\":\"vis\",\"value\":\"1\"},{\"key\":\"wgl\",\"value\":\"true\"},{\"key\":\"ca_type\",\"value\":\"image\"}]}},\"continuation\":"${this.continuation}"}`,
        "method": "POST",
        "mode": "cors",
        "credentials": "include"
        }).then(data=>data.json()).then(res => {
            this.videos = [];
            this.continuation = '';
            
            for(let continuation of res.onResponseReceivedActions[0].appendContinuationItemsAction.continuationItems){
                if(continuation.gridVideoRenderer){
                    this.videos.push(continuation.gridVideoRenderer.videoId);
                }else if(continuation.continuationItemRenderer){
                    this.continuation = continuation.continuationItemRenderer.continuationEndpoint.continuationCommand.token;
                }
            }
        });
    },
    // 把模擬捲動獲得的影片加入播放清單
    addStoredVideos(){
        this.addVideoToList(this.videos);
        this.videos = [];
    },
}

上面那段程式，把 key、playlist、authorization 改成自己從 fetch 裡面得到的資料就可以使用了。可以在執行前修改，或者執行後輸入 LibbyYoutubeFunc.key 修改 key 等資料，之後就可以用 LibbyYoutubeFunc.getVideos()、LibbyYoutubeFunc.addVideos(10, 20)執行包裝好的程式，不過好像不能一次加入太多影片，所以我預設一次 30 部影片

結語
這篇算是進階的 JS 和開發者工具運用，雖然沒有複雜的操作，但是如果沒有一點程式碼基礎，可能還是不好理解。如果跳過包裝程式碼那段，加入播放清單其實就是：
開發者工具 -> 複製 fetch -> 修改影片 ID -> 執行

爬蟲技術探索：批次將Youtube影片加入播放清單

前言

操作方式

結語