匿踪库卡隆,4001108003,kwill 灿烈
目录
目标:https://hr.tencent.com/position.php?&start=0#a
爬取所有的职位信息信息
class tecentjobitem(scrapy.item): # define the fields for your item here like: positionname = scrapy.field() positionlink = scrapy.field() positiontype = scrapy.field() peoplenum = scrapy.field() worklocation = scrapy.field() publishtime = scrapy.field()
# -*- coding: utf-8 -*- import scrapy from tecentjob.items import tecentjobitem class tencentspider(scrapy.spider): name = 'tencent' allowed_domains = ['tencent.com'] url = 'https://hr.tencent.com/position.php?&start=' offset = 0 start_urls = [url + str(offset)] def parse(self, response): for each in response.xpath("//tr[@class = 'even'] | //tr[@class = 'odd']"): # 初始化模型对象 item = tecentjobitem() item['positionname'] = each.xpath("./td[1]/a/text()").extract()[0] item['positionlink'] = each.xpath("./td[1]/a/@href").extract()[0] item['positiontype'] = each.xpath("./td[2]/text()").extract()[0] item['peoplenum'] = each.xpath("./td[3]/text()").extract()[0] item['worklocation'] = each.xpath("./td[4]/text()").extract()[0] item['publishtime'] = each.xpath("./td[5]/text()").extract()[0] yield item if self.offset < 100: self.offset += 10 # 将请求重写发送给调度器入队列、出队列、交给下载器下载 # 拼接新的rurl,并回调parse函数处理response # yield scrapy.request(url, callback = self.parse) yield scrapy.request(self.url + str(self.offset), callback=self.parse)
import json class tecentjobpipeline(object): def __init__(self): self.filename = open("tencent.json", 'wb') def process_item(self, item, spider): text = json.dumps(dict(item),ensure_ascii=false) + "\n" self.filename.write(text.encode('utf-8')) return item def close_spider(self, spider): self.filename.close()
default_request_headers = { 'user-agent': 'mozilla/5.0 (windows nt 10.0; win64; x64) applewebkit/537.36 (khtml, like gecko) chrome/71.0.3578.98 safari/537.36', 'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', 'accept-language': 'en', }
如对本文有疑问,请在下面进行留言讨论,广大热心网友会与你互动!! 点击进行留言回复
Python爬虫:Request Payload和Form Data的简单区别说明
浅谈Python中threading join和setDaemon用法及区别说明
Python3-异步进程回调函数(callback())介绍
python继承threading.Thread实现有返回值的子类实例
Python中使用threading.Event协调线程的运行详解
网友评论