1. 使用excel（简单使用）
数据- 自网站-导入
2.you-get
python爬虫入门
1.环境配置
python，request，lxml
2.原理
爬虫的框架如下：
1.挑选种子URL；
2.将这些URL放入待抓取的URL队列；3.取出待抓取的URL，下载并存储进已下载网页库中。此外，将这些URL放入待抓取URL队列，进入下一循环；4.分析已抓取队列中的URL，并且将URL放入待抓取URL队列，从而进入下一循环。
爬虫的基本流程：
简单的说，我们向服务器发送请求后，会得到返回的页面，通过解析页面之后，我们可以抽取我们想要的那部分信息，并存储在指定的文档或数据库中。这样，我们想要的信息就被我们“爬”下来啦~
Requests+Xpath爬去豆瓣电影
import requests from lxml import etree from bs4 import BeautifulSoup def movie_spider(url): data = requests.get(url).text s = etree.HTML(data) # 根据页面的xpath解析数据 film = s.xpath('/html/body/div[3]/div[1]/h1/span[1]/text()') director = s.xpath('/html/body/div[3]/div[1]/div[3]/div[1]/div[1]/div[1]/div[1]/div[2]/span[1]/span[2]/a/text()') # 浏览器检查的Xpath和实际不对应？为什么？ actor1= s.xpath('//*[@id="info"]/span[3]/span[2]/a[1]/text()') time = s.xpath('/html/body/div[3]/div[1]/div[3]/div[1]/div[1]/div[1]/div[1]/div[2]/span[13]/text()') print(film,director,actor1,time) 静态网页抓取 def top250_movie_spider(): """ 1.定制请求头，network>user-agent :return: """ movie_list = [] headers = { 'user-agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:63.0) Gecko/20100101 Firefox/63.0', 'Host':'movie.douban.com' } for i in range(0,10): link = 'https://movie.douban.com/top250?start='+str(i*25)+'&filter=' r = requests.get(link,headers=headers,timeout=10) # 使用BeautifulSoup解析网页 soup = BeautifulSoup(r.text,'lxml') div_list = soup.find_all('div',class_='hd') for each in div_list: movie = each.a.span.text.strip() movie_list.append(movie) return movie_list if __name__ == "__main__": url = "https://movie.douban.com/subject/1292052/" # movie_spider(url) print(top250_movie_spider()) 动态网页抓取
解析真实地址抓取：
network（刷新网页抓包）>XHR>确定真实地址
json库解析数据
import requests import json link ="https://api-zero.livere.com/v1/comments/list?callback=jQuery1124047657659644175476_1543235317407&limit=10&repSeq=4272904&requestPath=%2Fv1%2Fcomments%2Flist&consumerSeq=1020&livereSeq=28583&smartloginSeq=5154&_=1543235317409" def single_page_comment(link): # 定制请求头 headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.89 Safari/537.36'} r = requests.get(link, headers=headers) json_string = r.text json_string = json_string[json_string.find('{'):-2] json_data = json.loads(json_string) common_list = json_data['results']['parents'] for eachone in common_list: message = eachone['content'] print(message) # 对比不同的地址，找出改变的量 for page in range(1,4): link1 = "https://api-zero.livere.com/v1/comments/list?callback=jQuery112403473268296510956_1531502963311&limit=10&offset=" link2 = "&repSeq=4272904&requestPath=%2Fv1%2Fcomments%2Flist&consumerSeq=1020&livereSeq=28583&smartloginSeq=5154&_=1531502963316" page_str = str(page) link = link1 + page_str + link2 print (link) single_page_comment(link) 使用Selenium模拟浏览器抓取
from selenium import webdriver from selenium.webdriver.firefox.firefox_binary import FirefoxBinary import time # # 需要geckodriver,并放入环境变量中 --错误 # driver = webdriver.Firefox() # driver.get("https://www.baidu.com") #动态爬取网页 caps = webdriver.DesiredCapabilities().FIREFOX caps["marionette"] = True # 计算机中firefox的地址 binary = FirefoxBinary(r"C:\Program Files (x86)\Mozilla Firefox\firefox.exe") driver = webdriver.Firefox(firefox_binary=binary,capabilities=caps) for i in range(0,20): driver.get("https://zh.airbnb.com/s/Shenzhen--China/homes?refinement_paths%5B%5D=%2Fhomes&click_referer=t%3ASEE_ALL%7Csid%3Ad66de168-dbd5-42e7-a122-bd4a001f781e%7Cst%3AMAGAZINE_HOMES&title_type=MAGAZINE_HOMES&query=Shenzhen%2C%20China&allow_override%5B%5D=&s_tag=ckaduMCI§ion_offset=4&items_offset="+str(i*18)) rent_list = driver.find_elements_by_css_selector('div._gig1e7') for eachhouse in rent_list: comment = eachhouse.find_element_by_css_selector('span._1cy09umr') comment = comment.text price_detail = eachhouse.find_elements_by_css_selector("span._1sfeueqe") price_list = [i.text for i in price_detail] price = price_list[1] name = eachhouse.find_element_by_css_selector("div._190019zr") name = name.text details = eachhouse.find_element_by_css_selector("div._1etkxf1") details = details.text print(comment,price,name,details)