爬虫利器Puppeteer实战

# 安装命令
npm i puppeteer --save

# 错误信息
ERROR: Failed to download Chromium r515411! Set "PUPPETEER_SKIP_CHROMIUM_DOWNLOAD" env variable to skip download.

# 设置环境变量跳过下载 Chromium
set PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=1 

# 或者可以这样干，只下载模块而不build
npm i --save puppeteer --ignore-scripts

# 成功安装模块
+ puppeteer@0.13.0
added 1 package in 1.77s

PUPPETEER_DOWNLOAD_HOST=https://storage.googleapis.com.cnpmjs.org
npm i puppeteer

npm install -g cnpm --registry=https://registry.npm.taobao.org
cnpm i puppeteer

const puppeteer = require('puppeteer');

(async () => {
 const browser = await (puppeteer.launch({
 // 若是手动下载的chromium需要指定chromium地址, 默认引用地址为 /项目目录/node_modules/puppeteer/.local-chromium/
 executablePath: '/Users/huqiyang/Documents/project/z/chromium/Chromium.app/Contents/MacOS/Chromium',
 //设置超时时间
 timeout: 15000,
 //如果是访问https页面 此属性会忽略https错误
 ignoreHTTPSErrors: true,
 // 打开开发者工具, 当此值为true时, headless总为false
 devtools: false,
 // 关闭headless模式, 不会打开浏览器
 headless: false
 }));
 const page = await browser.newPage();
 await page.goto('https://www.jianshu.com/u/40909ea33e50');
 await page.screenshot({
 path: 'jianshu.png',
 type: 'png',
 // quality: 100, 只对jpg有效
 fullPage: true,
 // 指定区域截图，clip和fullPage两者只能设置一个
 // clip: {
 // x: 0,
 // y: 0,
 // width: 1000,
 // height: 40
 // }
 });
 browser.close();
})();

const fs = require('fs');
const puppeteer = require('puppeteer');

(async () => {
 const browser = await (puppeteer.launch({ executablePath: '/Users/huqiyang/Documents/project/z/chromium/Chromium.app/Contents/MacOS/Chromium', headless: false }));
 const page = await browser.newPage();
 // 进入页面
 await page.goto('https://music.163.com/#');

 // 点击搜索框拟人输入 鬼才会想起
 const musicName = '鬼才会想';
 await page.type('.txt.j-flag', musicName, {delay: 0});

 // 回车
 await page.keyboard.press('Enter');

 // 获取歌曲列表的 iframe
 await page.waitFor(2000);
 let iframe = await page.frames().find(f => f.name() === 'contentFrame');
 const SONG_LS_SELECTOR = await iframe.$('.srchsongst');

 // 获取歌曲 鬼才会想起 的地址
 const selectedSongHref = await iframe.evaluate(e => {
 const songList = Array.from(e.childNodes);
 const idx = songList.findIndex(v => v.childNodes[1].innerText.replace(/\s/g, '') === '鬼才会想起');
 return songList[idx].childNodes[1].firstChild.firstChild.firstChild.href;
 }, SONG_LS_SELECTOR);

 // 进入歌曲页面
 await page.goto(selectedSongHref);

 // 获取歌曲页面嵌套的 iframe
 await page.waitFor(2000);
 iframe = await page.frames().find(f => f.name() === 'contentFrame');

 // 点击 展开按钮
 const unfoldButton = await iframe.$('#flag_ctrl');
 await unfoldButton.click();

 // 获取歌词
 const LYRIC_SELECTOR = await iframe.$('#lyric-content');
 const lyricCtn = await iframe.evaluate(e => {
 return e.innerText;
 }, LYRIC_SELECTOR);

 console.log(lyricCtn);

 // 截图
 await page.screenshot({
 path: '歌曲.png',
 fullPage: true,
 });

 // 写入文件
 let writerStream = fs.createWriteStream('歌词.txt');
 writerStream.write(lyricCtn, 'UTF8');
 writerStream.end();

 // 获取评论数量
 const commentCount = await iframe.$eval('.sub.s-fc3', e => e.innerText);
 console.log(commentCount);

 // 获取评论
 const commentList = await iframe.$$eval('.itm', elements => {
 const ctn = elements.map(v => {
 return v.innerText.replace(/\s/g, '');
 });
 return ctn;
 });
 console.log(commentList);
})();

const axios = require('axios');
const useAxios = () => {
 axios.get('https://www.guazi.com/hz/buy/')
 .then(((result) => {
 console.log(result.data);
 }))
 .catch((err) => {
 console.log(err);
 });
};

const fs = require('fs');
const puppeteer = require('puppeteer');

(async () => {
 const browser = await (puppeteer.launch({ executablePath: '/Users/huqiyang/Documents/project/z/chromium/Chromium.app/Contents/MacOS/Chromium', headless: true }));
 const page = await browser.newPage();

 // 进入页面
 await page.goto('https://www.guazi.com/hz/buy/');

 // 获取页面标题
 let title = await page.title();
 console.log(title);

 // 获取汽车品牌
 const BRANDS_INFO_SELECTOR = '.dd-all.clearfix.js-brand.js-option-hid-info';
 const brands = await page.evaluate(sel => {
 const ulList = Array.from($(sel).find('ul li p a'));
 const ctn = ulList.map(v => {
 return v.innerText.replace(/\s/g, '');
 });
 return ctn;
 }, BRANDS_INFO_SELECTOR);
 console.log('汽车品牌: ', JSON.stringify(brands));
 let writerStream = fs.createWriteStream('car_brands.json');
 writerStream.write(JSON.stringify(brands, undefined, 2), 'UTF8');
 writerStream.end();
 // await bodyHandle.dispose();

 // 获取车源列表
 const CAR_LIST_SELECTOR = 'ul.carlist';
 const carList = await page.evaluate((sel) => {
 const catBoxs = Array.from($(sel).find('li a'));
 const ctn = catBoxs.map(v => {
 const title = $(v).find('h2.t').text();
 const subTitle = $(v).find('div.t-i').text().split('|');
 return {
 title: title,
 year: subTitle[0],
 milemeter: subTitle[1]
 };
 });
 return ctn;
 }, CAR_LIST_SELECTOR);

 console.log(`总共${carList.length}辆汽车数据: `, JSON.stringify(carList, undefined, 2));

 // 将车辆信息写入文件
 writerStream = fs.createWriteStream('car_info_list.json');
 writerStream.write(JSON.stringify(carList, undefined, 2), 'UTF8');
 writerStream.end();

 browser.close();
})();