-
-
[原创]【原创】Python 二手车之家车辆档案数据爬虫
-
发表于: 2022-5-13 00:49
-
<table><tr><td bgcolor=orange>本文仅供学习交流使用，如侵立删！</td></tr></table>
二手车之家车辆档案数据爬虫
先上效果图
环境
- win10
- python3.9
- lxml、retrying、requests
需求分析
需求：
主要是需要车辆详情页中车辆档案的数据
先抓包分析一波，网页抓包没有什么有用的，转战APP
拿到数据接口就简单了，直接构造请求保存数据即可
获取车辆信息列表
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 | def _get_car_list(self, _url: str): """ 获取二手车信息列表 """ res = self._parse_url(_url=_url) ret = res.text # 解析获得字符串类型数据 result = etree.HTML(ret) # 转换数据类型为HTML,方便使用xpath url_list = result.xpath('//*[@id="goodStartSolrQuotePriceCore0"]/ul/li/a/@href') if not url_list: print('获取完成！') return for i in url_list: # 有些车型url直接是带域名的 if '5ebK9s2c8@1M7q4)9K6b7g2)9J5c8W2)9J5c8Y4N6%4N6#2)9J5k6h3y4Z5k6e0p5$3z5q4)9J5k6h3y4G2L8g2)9J5c8W2)9J5y4H3`.`. in i: yield 'https://' + i[2:] else: yield 'b0eK9s2c8@1M7s2y4Q4x3@1q4Q4x3V1k6Q4x3V1k6%4N6%4N6Q4x3X3g2U0K9r3f1I4y4U0S2Q4x3X3g2U0L8$3#2Q4x3U0M7`. + i |
获取车辆详情信息
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 | def _get_car_info(self, _url: str): """ 获取车辆详情信息 """ res = self._parse_url(_url=_url) ret = res.text # 解析获得字符串类型数据 result = etree.HTML(ret) # 转换数据类型为HTML,方便使用xpath # 标题 title = result.xpath('//div[@class="car-box"]/h3//text()') title = title[1].strip() if len(title) > 1 else title[0].strip() # 上牌时间 play_time = result.xpath('//*[@id="nav1"]/div[1]/ul[1]/li[1]/text()') play_time = play_time[0].strip() if play_time else '-' # 表显里程 display_mileage = result.xpath('//*[@id="nav1"]/div[1]/ul[1]/li[2]/text()') display_mileage = display_mileage[0].strip() if display_mileage else '-' # 变速箱 gearbox = result.xpath('//*[@id="nav1"]/div[1]/ul[1]/li[3]/text()') gearbox = gearbox[0].strip() if gearbox else '-' # 排放标准 emission_standards = result.xpath('//*[@id="nav1"]/div[1]/ul[1]/li[4]/text()') emission_standards = emission_standards[0].strip() if emission_standards else '-' # 排量 displacement = result.xpath('//*[@id="nav1"]/div[1]/ul[1]/li[5]/text()') displacement = displacement[0].strip() if displacement else '-' # 发布时间 release_time = result.xpath('//*[@id="nav1"]/div[1]/ul[1]/li[6]/text()') release_time = release_time[0].strip() if release_time else '-' # 年检到期 annual_inspection_expires = result.xpath('//*[@id="nav1"]/div[1]/ul[2]/li[1]/text()') annual_inspection_expires = annual_inspection_expires[0].strip() if annual_inspection_expires else '-' # 保险到期 insurance_expires = result.xpath('//*[@id="nav1"]/div[1]/ul[2]/li[2]/text()') insurance_expires = insurance_expires[0].strip() if insurance_expires else '-' # 质保到期 warranty_expires = result.xpath('//*[@id="nav1"]/div[1]/ul[2]/li[3]/text()') warranty_expires = warranty_expires[0].strip() if warranty_expires else '-' # 过户次数 number_of_transfers = result.xpath('//*[@id="nav1"]/div[1]/ul[2]/li[5]/text()') number_of_transfers = number_of_transfers[0].strip() if number_of_transfers else '-' # 所在地 location = result.xpath('//*[@id="nav1"]/div[1]/ul[2]/li[6]/text()') location = location[0].strip() if location else '-' # 发动机 engine = result.xpath('//*[@id="nav1"]/div[1]/ul[3]/li[1]/text()') engine = engine[0].strip() if engine else '-' # 车辆级别 vehicle = result.xpath('//*[@id="nav1"]/div[1]/ul[3]/li[2]/text()') vehicle = vehicle[0].strip() if vehicle else '-' # 车身颜色 car_color = result.xpath('//*[@id="nav1"]/div[1]/ul[3]/li[3]/text()') car_color = car_color[0].strip() if car_color else '-' # 燃油标号 fuel_label = result.xpath('//*[@id="nav1"]/div[1]/ul[3]/li[4]/text()') fuel_label = fuel_label[0].strip() if fuel_label else '-' # 驱动方式 drive_mode = result.xpath('//*[@id="nav1"]/div[1]/ul[3]/li[5]/text()') drive_mode = drive_mode[0].strip() if drive_mode else '-' data = [[title, play_time, display_mileage, gearbox, emission_standards, displacement, release_time, annual_inspection_expires, insurance_expires, warranty_expires, number_of_transfers, location, engine, vehicle, car_color, fuel_label, drive_mode, _url]] print(data) self._save_csv(data=data) |
资源下载
<table><tr><td bgcolor=orange>本文仅供学习交流使用，如侵立删！</td></tr></table>
[培训]科锐软件逆向54期预科班、正式班开始火爆招生报名啦！！！
