郑日宇个人资料,装修除甲醛最好方法,劫尽1.0c
根据规律手工构造各个期刊的url
爬取两遍，第一遍爬完，去重（集合），并初始化结果集；第二遍爬取，在结果集里查询并计数
将结果集转化为列表，并按照列表中times关键字对字典（相当于数据集合）整体排序
写入结果到txt文件
我要评论from bs4 import beautifulsoup as be
import requests as req
import os
baseurl = "http://www.52duzhe.com/"
def do_soup(url):
try:
r = req.get(url,headers={'user-agent':'mozilla/5.0'})
r.raise_for_status()
r.encoding = r.apparent_encoding
html = r.text
soup = be(html,'lxml')
return soup
except:
print("获取"+url+"失败")
def search_each_per(tag):
aut_set = set()
a_list = []
for i in range(2010,2018):
for j in range(1,25):
if j<10:
extraurl = str(i)+'_0'+str(j)
else:
extraurl = str(i)+'_'+str(j)
if i in [2010,2011,2012]:
if(i == 2012 and j>=14):
url = baseurl + extraurl + r"/"
else:
url = baseurl + '1_' + extraurl + ".html"
else:
url = baseurl + extraurl + r"/"
soup = do_soup(url) #使用函数
if soup == none:
continue
per_aut_list = soup.find_all('td',class_="author")
if tag==1:
for k in per_aut_list:
aut_set.add(k.string)
print("{}年{}期作者已入库".format(i,j))
else:
for k in per_aut_list:
a_list.append(k.string)
if tag==1:
return list(aut_set) #返回了一个去重后的作者列表
else:
return a_list #返回了一个有重复元素的列表，用于计数
def main():
author_list0 = search_each_per(1) # 1代表一个控制标记,接收无重列表
print("正在接收有重复数据列表,请等待...")
a_list = search_each_per(0) #接收有重复元素列表
result = {} #放结果的字典
for i in author_list0:
result[str(i)] = 0 #初始化统计结果
for j in a_list:
if i==j:
result[str(i)] += 1
#下面对结果按发表次数做降序处理
print("下面对结果按发表次数做降序处理...")
att = [] #做一个容器
for key,value in result.items():
j={}
j["author"]=key
j["times"]=value
att.append(j)
att.sort(key = lambda x:x["times"],reverse = true)
# 将结果写入text文本中
print("将结果写入text文本中,请耐心等待...")
path = os.getcwd()
filename = path + "读者作者结果1.txt"
new = open(filename,"w",errors='ignore') #网络字节流中可能有不合法字符要忽略：illegal multibyte sequence
for i in att:
author = i["author"]
times = i["times"]
print(author)
print(times)
if author == none: #unsupported operand type(s) for +: 'nonetype' and 'str'
new.write("none" +"\t" + str(times) + "\n")
else:
new.write(author +"\t" + str(times) + "\n")
new.close()
print("完成统计")
main()
根据规律手工构造各个期刊的url
爬取一遍，用列表（容器）放集合（数据集），并设置tag标签，判断列表中是否有此作者，没有就增加，有就 times+1
直接返回列表，并按照列表中times关键字对字典（相当于数据集合）整体排序
写入结果到txt文件
from bs4 import beautifulsoup as be
import requests as req
import os
baseurl = "http://www.52duzhe.com/"
def do_soup(url):
try:
r = req.get(url,headers={'user-agent':'mozilla/5.0'})
r.raise_for_status()
r.encoding = r.apparent_encoding
html = r.text
soup = be(html,'lxml')
return soup
except:
print("获取"+url+"失败")
def search_each_per():
obj_list = []
for i in range(2010,2018):
for j in range(1,25):
if j<10:
extraurl = str(i)+'_0'+str(j)
else:
extraurl = str(i)+'_'+str(j)
if i in [2010,2011,2012]:
if(i == 2012 and j>=14):
url = baseurl + extraurl + r"/"
else:
url = baseurl + '1_' + extraurl + ".html"
else:
url = baseurl + extraurl + r"/"
soup = do_soup(url) #使用函数
if soup == none:
continue
per_aut_list = soup.find_all('td',class_="author")
for it in per_aut_list: #别习惯性用i,换个名字，是个坑
tag = 0
for jk in obj_list:
if(jk["author"] == it.string):
jk["times"] += 1
tag = 1
break
if(tag == 0):
obj = {"author":it.string,"times":1}
obj_list.append(obj)
return obj_list
def main():
print("正在创建结果对象列表，请耐心等待...")
obj_list = search_each_per() #接受结果列表
#下面对结果按发表次数做降序处理
print("下面对结果按发表次数做降序处理...")
obj_list.sort(key = lambda x:x["times"],reverse = true)
# 将结果写入text文本中
print("将结果写入text文本中,请耐心等待...")
path = os.getcwd()
filename = path + "读者作者结果3.txt"
new = open(filename,"w",errors='ignore') #网络字节流中可能有不合法字符要忽略：illegal multibyte sequence
for i in obj_list:
author = i["author"]
times = i["times"]
print(author)
print(times)
if author == none: #unsupported operand type(s) for +: 'nonetype' and 'str'
new.write("none" +"\t" + str(times) + "\n")
else:
new.write(author +"\t" + str(times) + "\n")
new.close()
print("完成统计")
main()
使用类创建对象（数据集）
直接在主页爬取各期刊链接，放在列表里，遍历列表查询各期刊作者
爬取一遍，用列表（容器）放对象（数据集），并设置tag标签，判断列表中是否有此对象，没有就实例化一个对象并更新列表，有就 对象.times+1
直接返回列表，并按照列表中对象的times属性对对象（相当于数据集合）整体排序
写入结果到txt文件
class author(object):
def __init__(self,name):
self.name = name
self.times = 1
from bs4 import beautifulsoup as be
import requests as req
import os
baseurl = "http://www.52duzhe.com/"
def do_soup(url):
try:
r = req.get(url)
r.raise_for_status()
r.encoding = r.apparent_encoding
html = r.text
soup = be(html,'lxml')
return soup
except:
print("获取"+url+"失败")
def search_each_per():
url_list = []
obj_list = []
soup = do_soup(baseurl)
link = soup.select(".booklist a") #获得链接，放回字典
for item in link:
url = baseurl +item["href"]
url_list.append(url)
for url in url_list:
soup = do_soup(url) #使用函数
if soup == none:
continue
per_aut_list = soup.find_all('td',class_="author")
for i in per_aut_list:
tag = 0
for j in obj_list:
if(j.name == i.string):
j.times += 1
tag = 1
break
if(tag == 0):
obj = author(i.string)
obj_list.append(obj)
return obj_list
def main():
print("正在创建对象列表，请等待...........")
obj_list = search_each_per()
#下面对结果按发表次数做降序处理
print("下面对结果按发表次数做降序处理...")
obj_list.sort(key = lambda obj:obj.times,reverse = true)
# 将结果写入text文本中
print("将结果写入text文本中,请耐心等待...")
path = os.getcwd()
filename = path + "读者作者结果2.txt"
new = open(filename,"w",errors="ignore") #处理非法字符 illegal multibyte sequence
for i in obj_list:
author = i.name
times = i.times
print(author)
print(times)
if author == none:
new.write("none" +"\t" + str(times) + "\n")
else:
new.write(author +"\t" + str(times) + "\n")
new.close()
print("完成统计")
main()
如对本文有疑问，请在下面进行留言讨论，广大热心网友会与你互动！！ 点击进行留言回复
新手学习Python2和Python3中print不同的用法
Python基于os.environ从windows获取环境变量
网友评论