代码放在github上：https://github.com/cooljacket/SnackSpider，绕开需要js解析反爬虫的行为，下面贴张效果图：

var keyword = "d3.js";//@input(keyword, 查询关键字, 爬取该关键字搜索出来的京东商品) var comment_count = 100;//@input(comment_count, 爬取的评论数, 最多爬取多少条评论) var page_count = comment_count / 10; keyword = keyword.trim(); var scanUrls = []; scanUrls.push("http://search.jd.com/Search?keyword=" + keyword.replace(/ /g, "+") + "&enc=utf-8&scrolling=y&page=200"); var helperUrlRegexes = []; helperUrlRegexes.push("http://search\\.jd\\.com/Search\\?keyword=" + keyword.replace(/ /g, "\\+").replace(/\./g, "\\.") + "&enc=utf-8&scrolling=y&page=\\d+"); var configs = { domains: ["search.jd.com", "item.jd.com", "club.jd.com"], scanUrls: scanUrls, contentUrlRegexes: ["http://item\\.jd\\.com/\\d+.html"], helperUrlRegexes: helperUrlRegexes, interval: 10000, fields: [ { // 第一个抽取项 name: "title", selector: "//div[@id='name']/h1", required: true }, { // 第一个抽取项 name: "productid", selector: "//div[contains(@class,'fl')]/span[2]", required: true }, { name: "comments", selector: "//div[@id='comment-pages']/span", repeated: true, children: [ { name: "page", selector: "//text()" }, { name: "comments", sourceType: SourceType.AttachedUrl, attachedUrl: "http://club.jd.com/productpage/p-{$.productid}-s-0-t-3-p-{page}.html", selectorType: SelectorType.JsonPath, selector: "$.comments", repeated: true, children:[ { name: "com_content", selectorType: SelectorType.JsonPath, selector: "$.content" }, { name: "com_nickname", selectorType: SelectorType.JsonPath, selector: "$.nickname" } ] } ] } ] }; configs.afterDownloadPage = function(page, site) { var matches = /item\.jd\.com\/(\d+)\.html/.exec(page.url); if (!matches) return page; var commentUrl = "http://club.jd.com/productpage/p-"+matches[1]+"-s-0-t-3-p-0.html"; var result = site.requestUrl(commentUrl); var data = JSON.parse(result); var commentCount = data.productCommentSummary.commentCount; var pages = commentCount / 10; if (pages > page_count) pages = page_count; var pageHtml = "<div id=\"comment-pages\">"; for (var i = 0; i < pages; i++) { pageHtml += "<span>" + i + "</span>"; } pageHtml += "</div>"; var index = page.raw.indexOf("</body>"); page.raw = page.raw.substring(0, index) + pageHtml + page.raw.substring(index); return page; }; var dataSku = 0; configs.onProcessHelperPage = function(page, content, site) { var num = parseInt(extract(content, "//*[@id='J_goodsList']/ul/li[1]/@data-sku")); if (dataSku === 0) { dataSku = isNaN(num) ? 0 : num; } else if (dataSku === num) { dataSku = 0; return false; } var currentPageNum = parseInt(page.url.substring(page.url.indexOf("&page=") + 6)); if (currentPageNum === 0) { currentPageNum = 1; } var pageNum = currentPageNum + 1; var nextUrl = page.url.replace("&page=" + currentPageNum, "&page=" + pageNum); site.addUrl(nextUrl); return true; }; configs.afterExtractPage = function(page, data) { if (data.comments === null || data.comments === undefined) return data; var comments = []; for (var i = 0; i < data.comments.length; i++) { var p = data.comments[i]; for (var j = 0; j < p.comments.length; j++) { comments.push(p.comments[j]); } } data.comments = comments; return data; }; var crawler = new Crawler(configs); crawler.start(); //牛妹双11快乐，大家双11快乐(*^▽^*)

项目分析与小结： http://blog.csdn.net/zhyh1435589631/article/details/53053949 实现环境：  pyspider + centos7 + mysql 5.5 pyspider 部分代码： #!/usr/bin/env python # -*- encoding: utf-8 -*- # Created on 2016-11-05 23:18:55 # Project: taobao_food from pyspider.libs.base_handler import * import re import json import MySQLdb class Handler (BaseHandler): # 数据库链接配置 def __init__ (self): db_host= "127.0.0.1" user= "root" passwd= "zhyh2010" db= "taobao_food" charset= "utf8" conn = MySQLdb.connect(host=db_host, user = user, passwd=passwd, db=db, charset=charset) conn.autocommit( True ) self.db=conn.cursor() # 爬虫的起始url @every(minutes=24 * 60) def on_start (self): self.crawl( 'https://tce.taobao.com/api/mget.htm?callback=jsonp221&tce_sid=659631&tce_vid=8,2&tid=,&tab=,&topic=,&count=,&env=online,online' , callback=self.json_parser) # 解析相应的 json 数据 @config(age=24 * 60 * 60) def select_json (self, response): content = response.text pattern = re.compile( 'window.jsonp.*?\((.*?)\)' , re.S) content_select = re.findall(pattern, content) return content_select[ 0 ].strip() # 提取相应数据 插入数据库表中 def product_info (self, response): for data in response[ "result" ]: res = { "item_pic" : "https:" + data[ "item_pic" ], "item_youhui_price" : data[ "item_youhui_price" ], "item_title" : data[ "item_title" ] } sql = "insert into food_info(url, price, title) values (%s,%s,%s)" values = [(res[ "item_pic" ], res[ "item_youhui_price" ], res[ "item_title" ])] self.db.executemany(sql, values) # 解析 json @config(age=24 * 60 * 60) def json_parser (self, response): content = self.select_json(response) contents = json.loads(content) subres = contents[ "result" ] for each in contents[ "result" ]: info = self.product_info(subres[each])

也来凑个热闹吧，直接 python + selenium + chrome 来抓的，下面是结果+代码 #coding=utf-8 # 测试运行环境: ubuntu 14.04 python 2.7.6 chrome # 需要安装 selenim chromedriver from selenium import webdriver from selenium.webdriver.common.by import By import sys GBK = 'gbk' def logToFile(tag, msg): logFile = open('log', 'w') out = tag + '--\n' + msg logFile.write(out) def log(tag, msg): print tag, ' -- ' print msg def defLog(msg): log('out', msg) # 保存零食信息 class Item: def __init__(self): self.CODE = 'utf-8' # 输出内容到markdown文件中 class MarkdownWriter: def __init__(self, name='out.md'): mdFile = open(name, 'w') self.mdFile = mdFile def writeContent(self, content): self.mdFile.write(content) def writeItems(self, title, items): # 组装markdown格式 content = '### ' + title + ' \n' for item in items: content += '#### ' + item.title + ' \n' content += ' \n' content += '[goto](' + item.url + ') \n' content += 'money: ' + item.money + ' \n' content += 'store: ' + item.store + ' \n' content += '\n\n' self.mdFile.write(content) class TaoBaoSpider: def __init__(self): driver = webdriver.Chrome() self.driver = driver def getUrl(self, url): print 'start get ...' # 通过chrome加载url包括js脚本 self.driver.get(url) print 'get finished ...' def getHtmlWithJs(self): return self.driver.page_source def getElements(self): print 'get item ...' els = self.driver.find_elements(By.CSS_SELECTOR, "li[class=' item item-border'") return els def getContent(self, element): item = Item() # 从获取的html页面中获取需要的信息并封装到item类中 item.img = element.find_element_by_tag_name('img').get_attribute('src') item.money = element.find_element(By.CSS_SELECTOR, "div[class='price child-component clearfix'").find_element_by_tag_name('strong').text titleElement = element.find_element(By.CSS_SELECTOR, "div[class='title child-component'").find_element_by_class_name('J_AtpLog') item.title = titleElement.text item.url = titleElement.get_attribute('href') item.store = element.find_element(By.CSS_SELECTOR, "div[class='seller child-component clearfix'").find_element_by_tag_name('a').text return item def start(self, url): self.url = url self.getUrl(url) els = self.getElements() items = [] for e in els: item = self.getContent(e) items.append(item) return items def main(): # 设置下编码 reload(sys) sys.setdefaultencoding('utf-8') url = 'https://world.taobao.com/search/search.htm?_ksTS=1478358034370_312&spm=a21bp.7806943.20151106.1&search_type=0&_input_charset=utf-8&navigator=all&json=on&q=%E5%A5%B3%E6%9C%8B%E5%8F%8B%20%E9%9B%B6%E9%A3%9F&cna=Eg9NDplivkkCAXuCB323%2Fsy9&callback=__jsonp_cb&abtest=_AB-LR517-LR854-LR895-PR517-PR854-PR895' # 爬虫运行 spider = TaoBaoSpider() items = spider.start(url) # 输出到markdown文件中 writer = MarkdownWriter('taobao.md') writer.writeItems('零食列表', items) main() 更具体些的可以上这里看：https://github.com/5A59/Practice

没写爬淘宝的。老婆也不让买什么。写了个爬小说的。 package years.year2016.months11; import java.io.IOException; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.jsoup.select.Elements; import years.year2016.months10.WebUtil; public class WebDataGain { public static void main(String[]args){ WebDataGain w = new WebDataGain(); String url = "http://www.biqugezw.com/3_3096/"; String bookname = "一念永恒"; w.downNovel_Biqugezw(url,bookname); } /** * 下载笔趣阁小说功能 * @param url * @throws IOException */ public void downNovel_Biqugezw(String url,String bookName) { String url_root = "http://www.biqugezw.com"; //用Jsoup连接站点 Document doc=null; try { doc = Jsoup.connect(url).get(); } catch (IOException e2) { // TODO Auto-generated catch block e2.printStackTrace(); } //选择器，选择class做为容器 Elements elementList = doc.select("#list"); String query ="a[href~=/[0-9]{1}_[0-9]{4}/.*html]"; Elements elements = elementList.select(query); int size = elements.size(); System.out.println(size); String fileName = ""; int num = 0; int initnum=371; for(int i=initnum;i<size;i++){ Element e = elements.get(i); String href = e.attr("href"); String tempurl = url_root+href; System.out.println(tempurl); Document docInner=null; try { docInner = Jsoup.connect(tempurl).get(); } catch (IOException e1) { // TODO Auto-generated catch block e1.printStackTrace(); System.out.println(fileName); System.out.println(i); } Elements elementsClass = docInner.select(".bookname "); Elements elementsH = elementsClass.select("h1"); String sectionkname = elementsH.text(); System.out.println(sectionkname); Elements elementsContent = docInner.select("#content"); String content = elementsContent.text(); System.out.println(content); num=i%20; if(num==0&&i==0){ fileName="1-20章"; }else if(num==0&&i!=0){ fileName=i+"-"+(i+20)+"章节"; }else if(i==initnum){ int temp=initnum-num; fileName = temp+"-"+(temp+20)+"章节"; } try { WebUtil.downloadText(sectionkname+" "+content, bookName+"--"+fileName+".txt", WebUtil.getFileDir()+"//book//"+bookName+"//"); } catch (IOException e1) { // TODO Auto-generated catch block e1.printStackTrace(); } } } }

var keyword = "d3.js"; //@input(keyword, 查询关键字, 爬取该关键字搜索出来的京东商品) var comment_count = 100; //@input(comment_count, 爬取的评论数, 最多爬取多少条评论) var page_count = comment_count / 10; keyword = keyword.trim(); var scanUrls = []; scanUrls.push("http://search.jd.com/Search?keyword=" + keyword.replace(/ /g, "+") + "&enc=utf-8&scrolling=y&page=200"); var helperUrlRegexes = []; helperUrlRegexes.push("http://search\\.jd\\.com/Search\\?keyword=" + keyword.replace(/ /g, "\\+").replace(/\./g, "\\.") + "&enc=utf-8&scrolling=y&page=\\d+"); var configs = { domains: ["search.jd.com", "item.jd.com", "club.jd.com"], scanUrls: scanUrls, contentUrlRegexes: ["http://item\\.jd\\.com/\\d+.html"], helperUrlRegexes: helperUrlRegexes, interval: 10000, fields: [ { // 第一个抽取项 name: "title", selector: "//div[@id='name']/h1", required: true }, { // 第一个抽取项 name: "productid", selector: "//div[contains(@class,'fl')]/span[2]", required: true }, { name: "comments", selector: "//div[@id='comment-pages']/span", repeated: true, children: [ { name: "page", selector: "//text()" }, { name: "comments", sourceType: SourceType.AttachedUrl, attachedUrl: "http://club.jd.com/productpage/p-{$.productid}-s-0-t-3-p-{page}.html", selectorType: SelectorType.JsonPath, selector: "$.comments", repeated: true, children:[ { name: "com_content", selectorType: SelectorType.JsonPath, selector: "$.content" }, { name: "com_nickname", selectorType: SelectorType.JsonPath, selector: "$.nickname" } ] } ] } ] }; configs.afterDownloadPage = function(page, site) { var matches = /item\.jd\.com\/(\d+)\.html/.exec(page.url); if (!matches) return page; var commentUrl = "http://club.jd.com/productpage/p-"+matches[1]+"-s-0-t-3-p-0.html"; var result = site.requestUrl(commentUrl); var data = JSON.parse(result); var commentCount = data.productCommentSummary.commentCount; var pages = commentCount / 10; if (pages > page_count) pages = page_count; var pageHtml = "<div id=\"comment-pages\">"; for (var i = 0; i < pages; i++) { pageHtml += "<span>" + i + "</span>"; } pageHtml += "</div>"; var index = page.raw.indexOf("</body>"); page.raw = page.raw.substring(0, index) + pageHtml + page.raw.substring(index); return page; }; var dataSku = 0; configs.onProcessHelperPage = function(page, content, site) { var num = parseInt(extract(content, "//*[@id='J_goodsList']/ul/li[1]/@data-sku")); if (dataSku === 0) { dataSku = isNaN(num) ? 0 : num; } else if (dataSku === num) { dataSku = 0; return false; } var currentPageNum = parseInt(page.url.substring(page.url.indexOf("&page=") + 6)); if (currentPageNum === 0) { currentPageNum = 1; } var pageNum = currentPageNum + 1; var nextUrl = page.url.replace("&page=" + currentPageNum, "&page=" + pageNum); site.addUrl(nextUrl); return true; }; configs.afterExtractPage = function(page, data) { if (data.comments === null || data.comments === undefined) return data; var comments = []; for (var i = 0; i < data.comments.length; i++) { var p = data.comments[i]; for (var j = 0; j < p.comments.length; j++) { comments.push(p.comments[j]); } } data.comments = comments; return data; }; var crawler = new Crawler(configs); crawler.start(); //牛妹双11快乐，大家双11快乐(*^▽^*)

完全不会，只能照着抄一遍了。。

https://github.com/huowolf/practice/blob/master/JDSpider/src/com/huowolf/service/SpiderService.java

牛妹喜欢吃什么

首先。。。你要。。

什么时候可以写出这样的，明天回去试试

技术流都是通过技术找到女朋友的。哈哈

给个思路，phantomjs实现元搜索

var webpage = require('webpage') , page = webpage.create(); page.viewportSize = { width: 1024, height: 800 }; page.clipRect = { top: 0, left: 0, width: 1024, height: 800 }; page.settings = { javascriptEnabled: false, loadImages: true, userAgent: 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.31 (KHTML, like Gecko) PhantomJS/19.0' }; page.open('http://search.jd.com/Search?keyword=%E5%A5%B3%E6%9C%8B%E5%8F%8B%E9%9B%B6%E9%A3%9F&enc=utf-8&pvid=u4eabcvi.cns6qn', function (status) { var data; if (status === 'fail') { console.log('open page fail!'); } else { page.render('./test.png'); console.log('the mirrior page has saved '); } // release the memory page.close(); });

双十一已经结束，本期也为双十一话题，大家本期写的又非常棒，为了让大家都过一个不一样的双十一，所有本期参与的都送一份大礼包！大家继续努力，再接再厉，毕竟学习知识才是最重要的~~

正在学python，争取早日能像楼上的巨巨一样，写出自己想要的程序。加油！