node.js 抓取代理ip实例代码

/** 支持：node.js v7.9.0*/const cheerio=require('cheerio');const fetch =require('node-fetch');const Promise=require('bluebird');let mongoose=require('mongoose');Promise.promisifyAll(mongoose);let Schema=mongoose.Schema;mongoose.connect('mongodb://localhost:27017/ipproxypool');let IPpool=new Schema({  ip:{type:String,unique:true}})let Ipproxy=mongoose.model('IP',IPpool);function fetchUrl(url){  fetch(url,{    method:'get',    headers:{    }  })  .then(res=>res.text())  .then(body=>{    let $=cheerio.load(body);    let length=$('#list table tbody').find('tr').length;    for (let i=0;i<length;i++){    let ipaddress= $('#list table tbody').find('tr').eq(i).find('td').eq(0).text() ;    let port = $('#list table tbody').find('tr').eq(i).find('td').eq(1).text();    console.log(`IP:${ipaddress}:${port}`);    let ip=`${ipaddress}:${port}`    let ippool=new Ipproxy({      ip:ip    })    ippool.save();    }  })}var sleep = function (time) {  return new Promise(function (resolve, reject) {    setTimeout(function () {      resolve('ok');    }, time);  })};const pageNumber=10;var start = async function(){  for(let j=1;j<pageNumber;j++){     console.log(`当前是第${j}次等待..`);    fetchUrl(`http://www.kuaidaili.com/free/inha/${j}/`);    await sleep(1500);  }}start();

{ "name": "demo-4-ipproxypool", "version": "1.0.0", "description": "", "main": "index.js", "scripts": {  "test": "echo /"Error: no test specified/" && exit 1" }, "author": "false-l", "license": "", "devDependencies": {  "babel-preset-es2015": "^6.24.1",  "babel-preset-react": "^6.24.1",  "babel-preset-stage-3": "^6.24.1" }, "dependencies": {  "babel-core": "^6.24.1",  "bluebird": "^3.5.0",  "cheerio": "^0.22.0",  "koa": "^2.2.0",  "koa-router": "^7.1.1",  "mongoose": "^4.9.6",  "node-fetch": "^1.6.3" }}

const Promise=require('bluebird');let mongoose=require('mongoose');const koa=require('koa');const app=new koa();var router = require('koa-router')();Promise.promisifyAll(mongoose);let Schema=mongoose.Schema;mongoose.connect('mongodb://localhost:27017/ipproxypool');let IPpool=new Schema({  ip:{type:String,unique:true}})let Ipproxy=mongoose.model('IP',IPpool);app.use(async (ctx, next) => { await next(); var data=await Ipproxy.find({},function(err,ips){  var ipmap=[];   ips.forEach(function(ip){     ipmap[ip._id]=ip;     //console.log(ip)   }); }) var map=data.map(ip=>ip.ip); ctx.response.type = 'text/json'; ctx.response.body = map;});app.listen(3000);console.log('server listen:3000')