挑战练习微博搜索爬取好课加微ypkc

1.如何登录2.如何按条件高级搜索指定4.如何获取每条中的信息6.如何到csv文件中1.利用webdriver首页，人工实现登录操作，然后等待页面跳转2.GETURLGET3.selenium“end”4.csscss5.查找页面是否有“下一页”按钮，如果有则该地址，否则结束爬虫6.利用csv库，将文件为csv文件#coding:utf-8importtimeimportcsvimportosfromseleniumimportfromsele mon.keysimportKeys#CSV文件路径CSV_PATHweibo_ad.csv'#登录页面URLLOGIN_URL= #开启浏览器def#driver=webdriver.Chrome(executable_path='./chromedriver')driver=webdriver.Chrome()returndriver#生成GETURLdefquery(today): #滚动屏幕defscroll_down(driver):html_page=driver.find_element_by_tag_name('html')foriinrange(10):#获取页面信息def#cards_sel='#Pl_Official_MyProfileFeed21>div>div'cards_sel='#pl_weibo_direct>div>div.feed_lists.W_texta>div'cards=driver.find_elements_by_css_selector(cards_sel)info_list=forcardin ='div>div.WB_feed_detail.clearfix>dl>div>div.content.clearfix>div.feed_content.wbcon>a.W_texta.W_fb'content_sel='div>div.WB_feed_detail.clearfix>dl>div>div.content.clearfix>div.feed_content.wbcon>p' ='div>div.WB_feed_detail.clearfix>dl>div>div.content.clearfix>div.feed_from.W_textb>a.W_textb' ='div>div.WB_feed_detail.clearfix>dl>div>div.content.clearfix>div.feed_from.W_textb>a.W_textb' ='div>div.feed_action.clearfix>ul>li:nth-child(2)>a>span>em' ='div>div.feed_action.clearfix>ul>li:nth-child(3)>a>span>

='div>div.feed_action.clearfix>ul>li:nth-child(4)>a>span>em'name=card.find_element_by_css_selector(name_sel).textcontent=card.find_element_by_css_selector(content_sel).texttime=card.find_element_by_css_selector(time_sel).textlink=card.find_element_by_css_selector(link_sel).get_attribute('href')relay=card.find_element_by_css_selector(relay_sel).textcomm=card.find_element_by_css_selector(comm_sel).textlike=card.find_element_by_css_selector(like_sel).textinfo_list.append([name,content,time,link,relay,comm,like])returninfo_list#翻页defnext_sel='#pl_weibo_direct>div>div.WB_cardwrap.S_bg2.relative>div>a'next_page=driver.find_elements_by_css_selector(next_sel)ifreturnnext_page[0].get_attribute('href')#保存数据到CSV文件defifutf-8#在打开csvutf-8，否则可能会显示乱码withopen(CSV_PATH,'a',encoding='utf-8')asf:writer=csv.writer(f)

withopen(CSV_PATH,'w+',encoding='utf-8')asf:writer=csv.writer(f)CSVwriter.writerow(['用户名','内容','时间','','转发量','评论量',#登录页面deflogin(driver,url):#启动爬虫的函数defrun_crawler(driver,url):info=find_cards_info(driver)next_page=find_next(driver)ifnext_page:run_cr