2장 useragent를 이용하여 한탭으로 크롤링

const parse = require("csv-parse/lib/sync");
const stringify = require("csv-stringify/lib/sync");
const fs = require("fs");
const puppeteer = require("puppeteer");

const csv = fs.readFileSync("csv/data.csv");
const records = parse(csv.toString("utf-8"));
//csv의 형식이 버퍼(0,1로 이루어진 컴퓨터 친화적인 데이터)임 > 문자열로 바꿈
//csv-parse의 parse메서드가 문자열을 2차원 배열로 바꿔줌

const crawler = async () => {
    try{
        const result = [];
        const browser =  await puppeteer.launch({headless : false});
        const page = await browser.newPage(); 
        //사람은 보통 한페이지를 열어서 다음페이지 감 (봇이 아닌 작업)
        
        await page.setUserAgent("Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.90 Mobile Safari/537.36");
        //나의 크롬브라우저로 접속한것처럼 사랑흉내를 냄(navigator.userAgent로확인!)  
        
        //그래서 10번을 넘나듬
        for(const [i,r] of records.entries()){ 
            await page.goto(r[1]);
            
            console.log(await page.evaluate("navigator.userAgent"));
            
            const text = await page.evaluate(()=>{ //개중요!!!!!!!!!!!!!!!!! evaluate를 받음!!!!!!!!!!!!!!
                const score = document.querySelector(".score.score_left .star_score");

                if(score){
                       return score.textContent;
                }
            });
                
            if(text){
                console.log(r[0],"평점",text.trim());
                result[i] = [r[0],r[1],text.trim()]; //result[i] async(i)를 통해 순서대로 2차원배열생성!!
            }

            await page.waitForTimeout(2000); //어떤 웹에서 자동화작업을 막을수도있으므로 시간차!!           
        }
        await page.close();
        await browser.close();
        const str = stringify(result);
        fs.writeFileSync("csv/result.csv",str);
        //csv-stringify의 stringify메서드가 2차원배열을 다시 문자열로 바꿔줌

    }catch(e){
        console.error(e);
    }
    
};

crawler();
인프런 커뮤니티 질문&답변
