인프런 커뮤니티 질문&답변

const puppeteer = require('puppeteer');
const dotenv = require('dotenv');

dotenv.config();

const crawler = async () => {
  try {
   
    const browser = await puppeteer.launch({ headless: false, args: ['--window-size=1920,1080', '--disable-notifications'] });
    const page = await browser.newPage();
    await page.setViewport({
      width: 1080,
      height: 1080,
    });
    await page.goto('https://search.naver.com/search.naver?where=nexearch&sm=top_hty&fbm=0&ie=utf8&query=서울+지게차');
    
    await page.waitFor(3000);
    await page.evaluate(() => { 
      window.scrollBy(0, 1500);
      document.querySelector('#loc-main-section-root > section > div > div.api_more_wrap > a').click();
    });
    await page.waitFor(3000);

    await page.waitFor(3000);
    await page.evaluate(() => { 
      document.querySelector('#_pcmap_list_scroll_container > ul > li:nth-child(1) > div.qbGlu > div.ouxiq.icT4K > div > div > span > a > span.hClKF').click();
    });
    await page.waitFor(3000);
    await page.evaluate(() => { 
      document.querySelector('#_pcmap_list_scroll_container > ul > li:nth-child(1) > div.qbGlu > div.ouxiq.icT4K > div > div > div > div:nth-child(1) > span.RUjqr > a').click();
    });
  } catch (e) {
    console.error(e);
  }
};

crawler();

Error: Evaluation failed: TypeError: Cannot read properties of null (reading 'click')

at __puppeteer_evaluation_script__:2:151

at ExecutionContext._evaluateInternal (/Users/youssi/Downloads/nodejs-crawler-master/lecture/node_modules/puppeteer/lib/ExecutionContext.js:122:13)

at processTicksAndRejections (node:internal/process/task_queues:96:5)

at async ExecutionContext.evaluate (/Users/youssi/Downloads/nodejs-crawler-master/lecture/node_modules/puppeteer/lib/ExecutionContext.js:48:12)

at async crawler (/Users/youssi/Downloads/nodejs-crawler-master/lecture/index.js:53:5)

-- ASYNC --

at ExecutionContext.<anonymous> (/Users/youssi/Downloads/nodejs-crawler-master/lecture/node_modules/puppeteer/lib/helper.js:111:15)

at DOMWorld.evaluate (/Users/youssi/Downloads/nodejs-crawler-master/lecture/node_modules/puppeteer/lib/DOMWorld.js:112:20)

-- ASYNC --

at Frame.<anonymous> (/Users/youssi/Downloads/nodejs-crawler-master/lecture/node_modules/puppeteer/lib/helper.js:111:15)

at Page.evaluate (/Users/youssi/Downloads/nodejs-crawler-master/lecture/node_modules/puppeteer/lib/Page.js:833:43)

at Page.<anonymous> (/Users/youssi/Downloads/nodejs-crawler-master/lecture/node_modules/puppeteer/lib/helper.js:112:23)

at crawler (/Users/youssi/Downloads/nodejs-crawler-master/lecture/index.js:53:16)

에러가 납니다. 구글 개발자도구에서 select an element~~~ 누르고 나서야

document.querySelector('#_pcmap_list_scroll_container > ul > li:nth-child(1) > div.qbGlu > div.ouxiq.icT4K > div > div > span > a > span.hClKF').click();
    });

위 소스가 정상적으로 작동을 합니다.

찾아보니

Javascript를 사용하여 브라우저에서 렌더링하는 웹 페이지를 가져오려고 합니다. 즉, Http Request Node를 사용하여 수신하는 초기 응답에 현재 검색 중인 데이터가 포함되지 않습니다. 당신의 데이터는 자바스크립트 안에 로드되어 브라우저에 의해 html에 삽입될 것입니다.

이것과 비슷한 거 같습니다. 크롤링이 안되는 것일까요??