python+mongodb数据抓取详细介绍-MongoDB

def parser(sn,url):
  try:
    headers = {
      。。。。。。
      }
    r = requests.get(url, headers=headers,timeout=30)
    html = r.content
    soup = BeautifulSoup(html,"lxml")
    dt = {}
    #partno
    a = soup.find("meta",itemprop="mpn")
    if a:
      dt['partno'] = a['content']
    #manufacturer
    b = soup.find("meta",itemprop="manufacturer")
    if b:
      dt['manufacturer'] = b['content']
    #description
    c = soup.find("span",itemprop="description")
    if c:
      dt['description'] = c.get_text().strip()
    #price
    price = soup.find("table",class_="table table-condensed occalc_pa_table")
    if price:
      cost = {}
      for i in price.find_all('tr'):
        if len(i) > 1:
          td = i.find_all('td')
          key=td[0].get_text().strip().replace(',','')
          val=td[1].get_text().replace(u'\u20ac','').strip()
          if key and val:
            cost[key] = val
      if cost:
        dt['cost'] = cost
        dt['currency'] = 'EUR'
    #quantity
    d = soup.find("input",id="ItemQuantity")
    if d:
      dt['quantity'] = d['value']
    #specs
    e = soup.find("div",class_="row parameter-container")
    if e:
      key1 = []
      val1= []
      for k in e.find_all('dt'):
        key = k.get_text().strip().strip('.')
        if key:
          key1.append(key)
      for i in e.find_all('dd'):
        val = i.get_text().strip()
        if val:
          val1.append(val)
      specs = dict(zip(key1,val1))
    if specs:
      dt['specs'] = specs
      print dt
      
    if dt:
      db[table].update({'sn':sn},{'$set':dt})
      print str(sn) + ' insert successfully'
      time.sleep(3)
    else:
      error(str(sn) + '\t' + url)
  except Exception,e:
    error(str(sn) + '\t' + url)
    print "Don't data!"

最后全部程序运行，将价值数据分析处理并存入数据库中！

以上就是本文关于python+mongodb数据抓取详细介绍的全部内容，希望对大家有所帮助。感兴趣的朋友可以继续参阅本站：Python探索之创建二叉树、Python探索之修改Python搜索路径、浅谈python中copy和deepcopy中的区别等，有什么问题，欢迎留言一起交流讨论。