史上最最最最最最最最全Python爬虫总结

 1 #coding=utf-8
 2 #python学习交流群：548377875
 3 import urllib  
 4 import urllib2  
 5 url = 'http://www.dataanswer.top'  
 6 headers = { 
 7     'host':'www.dataanswer.top',
 8     'user-agent':'mozilla/5.0 (x11; ubuntu; linux i686; rv:31.0) gecko/20100101 firefox/31.0',
 9     #'accept':'application/json, text/javascript, */*; q=0.01',
10     #'accept-language':'zh-cn,zh;q=0.8,en-us;q=0.5,en;q=0.3',
11     #'accept-encoding':'gzip,deflate',
12     #'referer':'http://www.dataanswer.top'
13 }   
14 request = urllib2.request(url,headers=headers)  
15 response = urllib2.urlopen(request)  
16 page = response.read()
17 print page

 1 #coding=utf-8
 2 import urllib  
 3 import urllib2  
 4 import os
 5 def getpage(url):     
 6     request = urllib2.request(url)  
 7         response = urllib2.urlopen(request)  
 8         return response.read()  
 9  
10  
11 url='http://www.dataanswer.top/'  
12 result=getpage(url)  
13 file_name='test.doc'
14 file_path='doc'
15 if os.path.exists(file_path) == false:
16     os.makedirs(file_path)
17 local=os.path.join(file_path,file_name)
18 f = open(local,"w+")  
19 f.write(result) 
20 f.close()
21  
22  
23 #coding=utf-8
24 import urllib  
25 import urllib2  
26 import os
27 def getpage(url):     
28     request = urllib2.request(url)  
29         response = urllib2.urlopen(request)  
30         return response.read()  
31  
32  
33 url='http://www.dataanswer.top/'  #把该地址改成图片/文件/视频/网页的地址即可
34 result=getpage(url)  
35 file_name='test.doc'
36 file_path='doc'
37 if os.path.exists(file_path) == false:
38     os.makedirs(file_path)
39 local=os.path.join(file_path,file_name)
40 urllib.urlretrieve(local)

 1 import urllib
 2 import urllib2
 3 import cookielib
 4  
 5 filename = 'cookie.txt'
 6 #声明一个mozillacookiejar对象实例来保存cookie，之后写入文件
 7 cookie = cookielib.mozillacookiejar(filename)
 8 opener = urllib2.build_opener(urllib2.httpcookieprocessor(cookie))
 9 postdata = urllib.urlencode({
10 'name':'春天里',
11 'pwd':'1222222'
12 })
13 #登录的url
14 loginurl = 'http://www.dataanswer.top/loginservice?action=tologin'
15 #模拟登录，并把cookie保存到变量
16 result = opener.open(loginurl,postdata)
17 #保存cookie到cookie.txt中
18 cookie.save(ignore_discard=true, ignore_expires=true)
19 #利用cookie请求访问另一个网址
20 gradeurl = 'http://www.dataanswer.top/loginservice?action=myhome'
21 #请求访问
22 result = opener.open(gradeurl)
23 print result.read()

 1 #coding=utf-8
 2 import sys, time, os, re
 3 import urllib, urllib2, cookielib
 4 loginurl = 'https://www.douban.com/accounts/login'
 5 cookie = cookielib.cookiejar()
 6 opener = urllib2.build_opener(urllib2.httpcookieprocessor(cookie))
 7 params = {
 8 "form_email":"13161055481",
 9 "form_password":"wwwwwww",
10 "source":"index_nav" #没有的话登录不成功
11 }
12 #从首页提交登录
13 response=opener.open(loginurl)
14 #验证成功跳转至登录页
15 print(response.geturl())
16 if response.geturl() == "https://www.douban.com/accounts/login":
17       html=response.read()
18     print(html)
19       #验证码图片地址--图片地址加密怎么办？？？
20       imgurl=re.search('<img id="captcha_image" src="(.+?)" alt="captcha" class="captcha_image"/>', html)
21     print(imgurl)
22       if imgurl:
23             url=imgurl.group(1)
24             #将图片保存至同目录下
25             res=urllib.urlretrieve(url,'v.jpg')
26             #获取captcha-id参数
27             captcha=re.search('<input type="hidden" name="captcha-id" value="(.+?)"/>',html)
28             if captcha:
29                   vcode=raw_input('请输入图片上的验证码：')
30                   params["captcha-solution"]=vcode
31                   params["captcha-id"]=captcha.group(1)
32                   params["user_login"]="登录"
33                   #提交验证码验证
34                   response=opener.open(loginurl, urllib.urlencode(params))
35                   ''' 登录成功跳转至首页 '''
36                   if response.geturl() == "https://www.douban.com/":
37                     print 'login success ! '
38                     print '准备进行发帖'
39                     addtopicurl="http://www.douban.com/group/python/new_topic"
40                     res=opener.open(addtopicurl)
41                     html=res.read()
42             else:
43                 print("fail3")
44         else:
45             print("fail2")
46     else:
47         print("fail1")
48 else:
49     print("fail0")

 1 #coding=utf-8
 2 #1、安装 python-pip
 3 #sudo apt-get install python-pip
 4 #2、安装selenium
 5 #sudo pip install -u selenium
 6  
 7  
 8 from selenium import webdriver
 9 driver = webdriver.firefox() 
10 driver.get('http://www.newsmth.net/nforum/#!article/intern/206790')
11 html=driver.page_source.encode('utf-8','ignore') #这个函数获取页面的html
12 print(html)
13 driver.close()

  1 #coding=utf-8
  2 """
  3 全网爬取所有链接，包括外链--广度优先
  4 """
  5 import urllib2
  6 import re
  7 from bs4 import beautifulsoup
  8 import time
  9  
 10  
 11 #爬虫开始的时间
 12 t=time.time()
 13 #设置的暂停爬取条数
 14 n_stop=10
 15  
 16  
 17 #存放已经爬取过的url
 18 checked_url=[]
 19 #存放待爬取的url
 20 checking_url=[]
 21 #存放连接失败的url
 22 fail_url=[]
 23 #存放不能连接的url
 24 error_url=[]
 25 #失败后允许连接的次数
 26 retry=3
 27 #连接超时时间
 28 timeout=20
 29  
 30  
 31 class url_node:
 32     def __init__(self,url):
 33         """
 34         url节点初始化
 35         ：param url:string 当前url
 36         """
 37         self.url=url
 38         self.content=''
 39  
 40  
 41     def __is_connectable(self):
 42         """
 43         检验url是否可以连接
 44         """
 45         #在允许连接次数下连接
 46         for i in range(retry):
 47             try:
 48                 #打开url没有报错，则表示可连接
 49                 response=urllib2.urlopen(self.url,timeout=timeout)
 50                 return true
 51             except:
 52                 #如果在尝试允许连接次数下报错，则不可连接
 53                 if i==retry-1:
 54                     return false
 55  
 56  
 57     def get_next(self):
 58         """
 59         获取爬取该页中包含的其他所有的url
 60         """
 61         soup=beautifulsoup(self.content)
 62         #******************在此处可以从网页中解析你想要的内容************************************
 63         next_urls=soup.findall('a')
 64         if len(next_urls)!=0:
 65             for link in next_urls:
 66                 tmp_url=link.get('href')
 67                 #如果url不在爬取过的列表中也不在待爬取列表中则把其放到待爬列表中（没有确保该url有效）
 68                 if tmp_url not in checked_url and tmp_url not in checking_url:
 69                     checking_url.append(tmp_url)
 70         
 71     def run(self):
 72         if self.url:
 73             if self.__is_connectable():
 74                 try:
 75                     #获取爬取页面的所有内容
 76                     self.content=urllib2.urlopen(self.url,timeout=timeout).read()
 77                     #从该页面中获取url
 78                     self.get_next()
 79  
 80  
 81                 except:
 82                     #把连接失败的存放起来
 83                     fail_url.append(self.url)
 84                     print('[!]connect failed')
 85             else:
 86                 #把不能连接的存放起来
 87                 error_url.append(self.url)
 88         else:
 89             print("所给的初始url有问题！")            
 90  
 91  
 92 if __name__=='__main__':
 93     #把初始的url放到待爬的列表中
 94     checking_url.append('http://www.36dsj.com/')
 95     #不断的从待爬的列表中获取url进行爬取
 96     ff=open("mytest.txt",'w')
 97     i=0
 98     for url in checking_url:
 99         #对该url进行爬取
100         url_node(url).run()
101         #存放已经爬取过的url
102         checked_url.append(url)    
103         #删除checking_url中已经爬取过的url
104         checking_url.remove(url)
105  
106  
107         i+=1
108         if i==n_stop:
109             #打出停止时的url，下次可以把该url作为初始继续
110             print url
111             print("爬取过的列表长度：%d") % len(checked_url)
112             print("待爬取的列表长度：%d") % len(checking_url)
113             print("连接失败的列表长度：%d") % len(fail_url)
114             print("不能连接的列表长度：%d") % len(error_url)
115             break
116     ff.close()
117     print("time:%d s") % (time.time()-t)    

  1 #coding=utf-8
  2 """
  3 爬取同一个网站所有的url,不包括外链
  4 """
  5 import urllib2
  6 import re
  7 from bs4 import beautifulsoup
  8 import time
  9  
 10 t=time.time()
 11  
 12 host=''
 13 checked_url=[]
 14 checking_url=[]
 15 result=[]
 16 retry=3
 17 timeout=20
 18  
 19 class url_node:
 20     def __init__(self,url):
 21         """
 22         url节点初始化
 23         ：param url:string 当前url
 24         """
 25         self.url=self.handle_url(url,is_next_url=false)
 26         self.next_url=[]
 27         self.content=''
 28  
 29  
 30     def handle_url(self,url,is_next_url=true):
 31         """
 32         将所有的url处理成标准形式
 33         """
 34         global checked_url
 35         global checking_url
 36  
 37         #去掉尾部的‘/’
 38         url=url[0:len(url)-1] if url.endswith('/') else url
 39  
 40         if url.find(host)==-1:
 41             if not url.startswith('http'):
 42                 url='http://'+host+url if url.startswith('/') else 'http://'+host+'/'+url
 43             else:
 44                 #如果含有http说明是外链，url的host不是当前的host，返回空
 45                 return
 46         else:
 47             if not url.startswith('http'):
 48                 url='http://'+url
 49  
 50  
 51         if is_next_url:
 52             #下一层url放入待检测列表
 53             if url not in checking_url:
 54                 checking_url.append(url)
 55         else:
 56             #对于当前需要检测的url将参数都替换为1，然后加入规则表
 57             #参数相同类型不同的url只检测一次
 58             rule=re.compile(r'=.*?\&|=.*?$')
 59             result=re.sub(rule,'=1&',url)
 60             if result in checked_url:
 61                 return '[!] url has checked!'
 62             else:
 63                 checked_url.append(result)
 64                 result.append(url)
 65         return url
 66  
 67  
 68     def __is_connectable(self):
 69         print("进入__is_connectable()函数")
 70         #检验是否可以连接
 71         retry=3
 72         timeout=2
 73         for i in range(retry):
 74             try:
 75                 #print("进入_..............函数")
 76                 response=urllib2.urlopen(self.url,timeout=timeout)
 77                 return true
 78             
 79             except:
 80                 if i==retry-1:
 81                     return false
 82  
 83  
 84     def get_next(self):
 85         #获取当前所有的url
 86         #print("进入get_next()函数")
 87         soup=beautifulsoup(self.content)
 88         next_urls=soup.findall('a')
 89         if len(next_urls)!=0:
 90             for link in next_urls:
 91                 self.handle_url(link.get('href'))
 92                 #print(link.text)
 93  
 94  
 95         
 96     def run(self):
 97         #print("进入run()函数")
 98         if self.url:
 99             #print self.url
100             if self.__is_connectable():
101                 try:
102                     self.content=urllib2.urlopen(self.url,timeout=timeout).read()
103                     self.get_next()
104  
105  
106                 except:
107                     print('[!]connect failed')
108 #处理https开头的url的类和方法
109 class poc:
110     def run(self,url):
111         global host
112         global checking_url
113         url=check_url(url)
114  
115  
116         if not url.find('https'):
117             host=url[:8]
118         else:
119             host=url[7:]
120  
121  
122         for url in checking_url:
123             print(url)
124             url_node(url).run()
125  
126  
127 def check_url(url):
128     url='http://'+url if not url.startswith('http') else url
129     url=url[0:len(url)-1] if url.endswith('/') else url
130  
131  
132     for i in range(retry):
133         try:
134             response=urllib2.urlopen(url,timeout=timeout)
135             return url
136         except:
137             raise exception("connect error")
138  
139  
140 if __name__=='__main__':
141     host='www.dataanswer.com'
142     checking_url.append('http://www.dataanswer.com/')
143     f=open('36大数据','w')
144     for url in checking_url:
145         f.write(url+'\n')
146         print(url)
147         url_node(url).run()
148     print result
149     print "url num:"+str(len(result))
150     print("time:%d s") % (time.time()-t)    

  1 #!/usr/bin/env python
  2 # -*- coding:utf-8 -*-
  3 """
  4 一个简单的python爬虫, 使用了多线程, 
  5 爬取豆瓣top前250的所有电影
  6 """
  7  
  8 import urllib2, re, string
  9 import threading, queue, time
 10 import sys
 11  
 12 reload(sys)
 13 sys.setdefaultencoding('utf8')
 14 _data = []
 15 file_lock = threading.lock()
 16 share_q = queue.queue()  #构造一个不限制大小的的队列
 17 _worker_thread_num = 3  #设置线程的个数
 18  
 19  
 20 class mythread(threading.thread) :
 21  
 22  
 23     def __init__(self, func) :
 24         super(mythread, self).__init__()  #调用父类的构造函数
 25         self.func = func  #传入线程函数逻辑
 26  
 27  
 28     def run(self) :
 29         self.func()
 30  
 31  
 32 def worker() :
 33     global share_q
 34     while not share_q.empty():
 35         url = share_q.get() #获得任务
 36         my_page = get_page(url)
 37         find_title(my_page)  #获得当前页面的电影名
 38         #write_into_file(temp_data)
 39         time.sleep(1)
 40         share_q.task_done()
 41  
 42  
 43 def get_page(url) :
 44     """
 45     根据所给的url爬取网页html
 46     args: 
 47         url: 表示当前要爬取页面的url
 48     returns:
 49         返回抓取到整个页面的html(unicode编码)
 50     raises:
 51         urlerror:url引发的异常
 52     """
 53     try :
 54         my_page = urllib2.urlopen(url).read().decode("utf-8")
 55     except urllib2.urlerror, e :
 56         if hasattr(e, "code"):
 57             print "the server couldn't fulfill the request."
 58             print "error code: %s" % e.code
 59         elif hasattr(e, "reason"):
 60             print "we failed to reach a server. please check your url and read the reason"
 61             print "reason: %s" % e.reason
 62     return my_page
 63  
 64  
 65 def find_title(my_page) :
 66     """
 67     通过返回的整个网页html, 正则匹配前100的电影名称
 68     args:
 69         my_page: 传入页面的html文本用于正则匹配
 70     """
 71     temp_data = []
 72     movie_items = re.findall(r'<span.*?class="title">(.*?)</span>', my_page, re.s)
 73     for index, item in enumerate(movie_items) :
 74         if item.find(" ") == -1 :
 75             #print item,
 76             temp_data.append(item)
 77     _data.append(temp_data)
 78  
 79 def main() :
 80     global share_q
 81     threads = []
 82     douban_url = "http://movie.douban.com/top250?start={page}&filter=&type="
 83     #向队列中放入任务, 真正使用时, 应该设置为可持续的放入任务
 84     for index in xrange(10) :   
 85         share_q.put(douban_url.format(page = index * 25))
 86     for i in xrange(_worker_thread_num) :
 87         thread = mythread(worker)
 88         thread.start()  #线程开始处理任务
 89     print("第%s个线程开始工作") % i
 90         threads.append(thread)
 91     for thread in threads :
 92         thread.join()
 93     share_q.join()
 94     with open("movie.txt", "w+") as my_file :
 95         for page in _data :
 96             for movie_name in page:
 97                 my_file.write(movie_name + "\n")
 98     print "spider successful!!!"
 99  
100  
101 if __name__ == '__main__':
102     main()