中国营销论坛,ca1569,monisa-za
本文实例讲述了python实现爬虫爬取nba数据功能。分享给大家供大家参考，具体如下：
爬取的网站为：stat-nba.com，这里爬取的是nba2016-2017赛季常规赛至2017年1月7日的数据
改变url_header和url_tail即可爬取特定的其他数据。
源代码如下：
我要评论
#coding=utf-8
import sys
reload(sys)
sys.setdefaultencoding('utf-8')
import requests
import time
import urllib
from bs4 import beautifulsoup
import re
from pyexcelerator import *
def geturllists(url_header,url_tail,pages):
"""
获取所有页面的url列表
"""
url_lists = []
url_0 = url_header+'0'+url_tail
print url_0
url_lists.append(url_0)
for i in range(1,pages+1):
url_temp = url_header+str(i)+url_tail
url_lists.append(url_temp)
return url_lists
def getnbaalldata(url_lists):
"""
获取所有2017赛季nba常规赛数据
"""
datasets = ['']
for item in url_lists:
data1 = getnbasingledata(item)
datasets.extend(data1)
#去掉数据里的空元素
for item in datasets[:]:
if len(item) == 0:
datasets.remove(item)
return datasets
def getnbasingledata(url):
"""
获取1个页面nba常规赛数据
"""
# url = 'http://stat-nba.com/query_team.php?querytype=game&order=1&crtcol=date_out&gametype=season&pagenum=3000&season0=2016&season1=2017'
# html = requests.get(url).text
html = urllib.urlopen(url).read()
# print html
soup = beautifulsoup(html)
data = soup.html.body.find('tbody').text
list_data = data.split('\n')
# with open('nba_data.txt','a') as fp:
# fp.write(data)
# for item in list_data[:]:
# if len(item) == 0:
# list_data.remove(item)
return list_data
def savedatatoexcel(datasets,sheetname,filename):
book = workbook()
sheet = book.add_sheet(sheetname)
sheet.write(0,0,u'序号')
sheet.write(0,1,u'球队')
sheet.write(0,2,u'时间')
sheet.write(0,3,u'结果')
sheet.write(0,4,u'主客')
sheet.write(0,5,u'比赛')
sheet.write(0,6,u'投篮命中率')
sheet.write(0,7,u'命中数')
sheet.write(0,8,u'出手数')
sheet.write(0,9,u'三分命中率')
sheet.write(0,10,u'三分命中数')
sheet.write(0,11,u'三分出手数')
sheet.write(0,12,u'罚球命中率')
sheet.write(0,13,u'罚球命中数')
sheet.write(0,14,u'罚球出手数')
sheet.write(0,15,u'篮板')
sheet.write(0,16,u'前场篮板')
sheet.write(0,17,u'后场篮板')
sheet.write(0,18,u'助攻')
sheet.write(0,19,u'抢断')
sheet.write(0,20,u'盖帽')
sheet.write(0,21,u'失误')
sheet.write(0,22,u'犯规')
sheet.write(0,23,u'得分')
num = 24
row_cnt = 0
data_cnt = 0
data_len = len(datasets)
print 'data_len:',data_len
while(data_cnt< data_len):
row_cnt += 1
print '序号:',row_cnt
for col in range(num):
# print col
sheet.write(row_cnt,col,datasets[data_cnt])
data_cnt += 1
book.save(filename)
def writedatatotxt(datasets):
fp = open('nba_data.txt','w')
line_cnt = 1
for i in range(len(datasets)-1):
#球队名称对齐的操作：如果球队名字过短或者为76人队是 球队名字后面加两个table 否则加1个table
if line_cnt % 24 == 2 and len(datasets[i]) < 5 or datasets[i] == u'费城76人':
fp.write(datasets[i]+'\t\t')
else:
fp.write(datasets[i]+'\t')
line_cnt += 1
if line_cnt % 24 == 1:
fp.write('\n')
fp.close()
if __name__ == "__main__":
pages = int(1132/150)
url_header = 'http://stat-nba.com/query_team.php?page='
url_tail = '&querytype=game&order=1&crtcol=date_out&gametype=season&pagenum=3000&season0=2016&season1=2017#label_show_result'
url_lists = geturllists(url_header,url_tail,pages)
datasets = getnbaalldata(url_lists)
writedatatotxt(datasets)
sheetname = 'nba normal data 2016-2017'
str_time = time.strftime('%y-%m-%d',time.localtime(time.time()))
filename = 'nba_normal_data'+str_time+'.xls'
savedatatoexcel(datasets,sheetname,filename)
更多关于python相关内容可查看本站专题：《python socket编程技巧总结》、《python正则表达式用法总结》、《python数据结构与算法教程》、《python函数使用技巧总结》、《python字符串操作技巧汇总》、《python入门与进阶经典教程》及《python文件与目录操作技巧汇总》
希望本文所述对大家python程序设计有所帮助。
如对本文有疑问，请在下面进行留言讨论，广大热心网友会与你互动！！ 点击进行留言回复
Python 实现将numpy中的nan和inf,nan替换成对应的均值
python爬虫把url链接编码成gbk2312格式过程解析
网友评论