python爬虫-Python二手车交易数据爬取

python爬⾍--Python⼆⼿车交易数据爬取通过Chrome浏览器登录数据来源页，使⽤Chrome的检查功能，多次刷新，先分析要爬取的数据的规律，搜索汽车类别、⼚商、品牌、车型、⾏驶⾥程、上牌⽇期、车⾝类型、燃油类型、变速箱、发动机功率、汽车有尚未修复的损坏、所在地区、报价类型、汽车售卖时间、⼆⼿车交易价格等。然后爬取所需页⾯的信息进⾏解析，将数据存储到数据库或者其它形式的⽂件。selenium打开⽹页#打开⼆⼿车⽹页并进⼊全部⼆⼿车出售页⾯#打开⼆⼿车⽹页并进⼊全部⼆⼿车出售页⾯fromseleniumimportwebdriverfromselenium.webdriver.supportimportexpected_conditionsasECfromselenium.webdriver.support.uiimportWebDriverWmon.byimportBydriver=webdriver.Chrome()driver.get('/guangzhou/')wait=WebDriverWait(driver,10)before=driver.current_window_handleconfirm_btn=wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR,'#tab2-2>div.btn-center>a')))confirm_btn.click()#点击查看更多打开详情页并爬取需要信息fullxpath：/html/body/div[12]/div[1]/ul/li[1]⼀共有56个即li[1]~li[56]driver.find_element_by_xpath(’/html/body/div[12]/div[1]/ul/li[1]’).click()点击进⼊详情页fullxpath：/html/body/div[12]/div[1]/ul/li[1]⼀共有56个即li[1]~li[56]driver.find_element_by_xpath(’/html/body/div[12]/div[1]/ul/li[1]’).click()点击进⼊详情页html=etreehtml=etree.HTML(driver.page_source,etree.HTMLPullParser())list_title=html.xpath('/html/body/div[5]/div[2]/h3//text()')#标题foriinrange(1,6):ifi==1:licheng.append(html.xpath('/html/body/div[5]/div[2]/ul/li[1]/h4/text()')[0])#⾥程数elifi==2:time.append(html.xpath('/html/body/div[5]/div[2]/ul/li[2]/h4/text()')[0])#上牌时间elifi==3:cangshu.append(html.xpath('/html/body/div[5]/div[2]/ul/li[3]/h4/text()')[0])#挡位/排量elifi==4:location.append(html.xpath('/html/body/div[5]/div[2]/ul/li[4]/h4/text()')[0])#车辆所在地elifi==5:str=html.xpath('/html/body/div[5]/div[2]/ul/li[5]/h4/text()')[0].strip()xianding.append(str)#限迁地个车辆信息foriinrange(1,57):#56个贩卖车辆driver.switch_to_window(driver.window_handles[1])#跳转while1:#防⽌太快窗⼝还没打开便转变窗⼝发⽣错误sleep(1)driver.find_element_by_xpath('//*[@class="viewlist_ul"]/li[{}]'.format(i)).click()driver.refresh()#刷新页⾯sleep(2)iflen(driver.window_handles)==3:#判断窗⼝是否打开driver.switch_to_window(driver.window_handles[2])breakelse:continueprint('爬取到第{}个'.format(i))html=etree.HTML(driver.page_source,etree.HTMLPullParser())#获取标题list_title=html.xpath('/html/body/div[5]/div[2]/h3//text()')str=list_title[0].strip()title=[str]price=[]#获取价格进⾏去除符号操作iflen(html.xpath('/html/body/div[5]/div[2]/div[2]/span//text()'))!=0:price=[(html.xpath('/html/body/div[5]/div[2]/div[2]/span//text()')[0]+'万').strip('¥')]eliflen(html.xpath('//*[@id="overlayPrice"]//text()'))!=0:price=[(html.xpath('//*[@id="overlayPrice"]//text()')[0]+'万').strip('￥')]else:price=[html.xpath('/html/body/div[5]/div[2]/div[2]/div[1]/div/text()')[0].strip('￥')]#print(price)#获取其他信息forjinrange(1,6):ifj==1:licheng=[]licheng=[(html.xpath('/html/body/div[5]/div[2]/ul/li[1]/h4/text()')[0])]elifj==2:time=[]time=[(html.xpath('/html/body/div[5]/div[2]/ul/li[2]/h4/text()')[0])]elifj==3:cangshu=[]cangshu=[(html.xpath('/html/body/div[5]/div[2]/ul/li[3]/h4/text()')[0])]elifj==4:location=[]location=[(html.xpath('/html/body/div[5]/div[2]/ul/li[4]/h4/text()')[0])]elifj==5:xianding=[]str=html.xpath('/html/body/div[5]/div[2]/ul/li[5]/h4/text()')[0].strip()xianding=[str]fromseleniumimportwebdriverfromseleniumimportwebdriverfromselenium.webdriver.supportimportexpected_conditionsasECfromselenium.webdriver.support.uiimportWebDriverWmon.byimportByfromlxmlimportetreeimportpandasaspdfromtimeimportsleep#proxy=['--proxy-server=4:9000']#chrome_options=webdriver.ChromeOptions()#chrome_options.add_argument(proxy[0])#driver=webdriver.Chrome(chrome_options=chrome_options)driver=webdriver.Chrome()driver.get('/guangzhou/')#打开⽹址wait=WebDriverWait(driver,10)before=driver.current_window_handleconfirm_btn=wait.until(#等待可以点击EC.element_to_be_clickable(EC.element_to_be_clickable((By.CSS_SELECTOR,'#tab2-2>div.btn-center>a')))confirm_btn.click()#点击’查看更多‘driver.switch_to_window(driver.window_handles[1])driver.find_element_by_xpath('//*[@id="currengpostion"]/div/div/ul[2]/li[1]/a').click()#点击默认排序b=0#作为是否是第⼀次写⼊的判断变量forlinrange(2,101):#100页driver.switch_to_window(driver.window_handles[1])#跳转窗⼝sleep(1)#等待ifl<=7:driver.find_element_by_xpath('//*[@id="listpagination"]/a[{}]'.format(l)).click()#点击第⼏页sleep(1)print('爬取到第{}页'.format(l-1))ifl>=8:#第六页后翻页模式driver.find_element_by_xpath('//*[@id="listpagination"]/a[6]'.format(l)).click()sleep(1)foriinrange(1,57):#⼀共有56个贩卖车辆driver.switch_to_window(driver.window_handles[1])#跳转while1:#防⽌太快窗⼝还没打开便转变窗⼝发⽣错误sleep(1)driver.find_element_by_xpath('//*[@class="viewlist_ul"]/li[{}]'.format(i)).click()driver.refresh()#刷新页⾯sleep(2)iflen(driver.window_handles)==3:#判断窗⼝是否打开driver.switch_to_window(driver.window_handles[2])breakelse:continueprint('爬取到第{}个'.format(i))html=etree.HTML(driver.page_source,etree.HTMLPullParser())#获取标题list_title=html.xpath('/html/body/div[5]/div[2]/h3//text()')str=list_title[0].strip()title=[str]price=[]#获取价格进⾏去除符号操作iflen(html.xpath('/html/body/div[5]/div[2]/div[2]/span//text()'))!=0:price=[(html.xpath('/html/body/div[5]/div[2]/div[2]/span//text()')[0]+'万').strip('¥')]eliflen(html.xpath('//*[@id="overlayPrice"]//text()'))!=0:price=[(html.xpath('//*[@id="overlayPrice"]//text()')[0]+'万').strip('￥')]else:price=[html.xpath('/html/body/div[5]/div[2]/div[2]/div[1]/div/text()')[0].strip('￥')]#print(price)#获取其他信息forjinrange(1,6):ifj==1:licheng=[]licheng=[(html.xpath('/html/body/div[5]/div[2]/ul/li[1]/h4/text()')[0])]elifj==2:time=[]time=[(html.xpath('/html/body/div[5]/div[2]/ul/li[2]/h4/text()')[0])]elifj==3:cangshu=[]cangshu=[(html.xpath('/html/body/div[5]/div[2]/ul/li[3]/h4/text()')[0])]elifj==4:location=[]location=[(html.xpath('/html/body/div[5]/div[2]