本文实例讲述了python实现爬取千万淘宝商品的方法。分享给大家供大家参考。具体实现方法如下：

import time
import leveldb
from urllib.parse import quote_plus 
import re
import json
import itertools
import sys
import requests
from queue import Queue
from threading import Thread
URL_BASE = \'http://s.m.taobao.com/search?q={}&n=200&m=api4h5&style=list&page={}\'
def url_get(url):
  # print(\'GET \' + url)
  header = dict()
  header[\'Accept\'] = \'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\'
  header[\'Accept-Encoding\'] = \'gzip,deflate,sdch\'
  header[\'Accept-Language\'] = \'en-US,en;q=0.8\'
  header[\'Connection\'] = \'keep-alive\'
  header[\'DNT\'] = \'1\'
  #header[\'User-Agent\'] = \'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1500.71 Safari/537.36\'
  header[\'User-Agent\'] = \'Mozilla/12.0 (compatible; MSIE 8.0; Windows NT)\'
  return requests.get(url, timeout = 5, headers = header).text
def item_thread(cate_queue, db_cate, db_item):
  while True:
    try:
      cate = cate_queue.get()
      post_exist = True
      try:
        state = db_cate.Get(cate.encode(\'utf-8\'))
        if state != b\'OK\': post_exist = False
      except:
        post_exist = False
      if post_exist == True:
        print(\'cate-{}: {} already exists ... Ignore\'.format(cate, title))
        continue
      db_cate.Put(cate.encode(\'utf-8\'), b\'crawling\')
      for item_page in itertools.count(1):
        url = URL_BASE.format(quote_plus(cate), item_page)
        for tr in range(5):
          try:
            items_obj = json.loads(url_get(url))
            break
          except KeyboardInterrupt:
            quit()
          except Exception as e:
            if tr == 4: raise e
        if len(items_obj[\'listItem\']) == 0: break
        for item in items_obj[\'listItem\']:
          item_obj = dict(
            _id = int(item[\'itemNumId\']),
            name = item[\'name\'],
            price = float(item[\'price\']),
            query = cate,
            category = int(item[\'category\']) if item[\'category\'] != \'\' else 0,
            nick = item[\'nick\'],
            area = item[\'area\'])
          db_item.Put(str(item_obj[\'_id\']).encode(\'utf-8\'),
                json.dumps(item_obj, ensure_ascii = False).encode(\'utf-8\'))
        print(\'Get {} items from {}: {}\'.format(len(items_obj[\'listItem\']), cate, item_page))
        if \'nav\' in items_obj:
          for na in items_obj[\'nav\'][\'navCatList\']:
            try:
              db_cate.Get(na[\'name\'].encode(\'utf-8\'))
            except:
              db_cate.Put(na[\'name\'].encode(\'utf-8\'), b\'waiting\')
      db_cate.Put(cate.encode(\'utf-8\'), b\'OK\')
      print(cate, \'OK\')
    except KeyboardInterrupt:
      break
    except Exception as e:
      print(\'An {} exception occured\'.format(e))
def cate_thread(cate_queue, db_cate):
  while True:
    try:
      for key, value in db_cate.RangeIter():
        if value != b\'OK\':
          print(\'CateThread: put {} into queue\'.format(key.decode(\'utf-8\')))
          cate_queue.put(key.decode(\'utf-8\'))
      time.sleep(10)
    except KeyboardInterrupt:
      break
    except Exception as e:
      print(\'CateThread: {}\'.format(e))
if __name__ == \'__main__\':
  db_cate = leveldb.LevelDB(\'./taobao-cate\')
  db_item = leveldb.LevelDB(\'./taobao-item\')
  orig_cate = \'正装\'
  try:
    db_cate.Get(orig_cate.encode(\'utf-8\'))
  except:
    db_cate.Put(orig_cate.encode(\'utf-8\'), b\'waiting\')
  cate_queue = Queue(maxsize = 1000)
  cate_th = Thread(target = cate_thread, args = (cate_queue, db_cate))
  cate_th.start()
  item_th = [Thread(target = item_thread, args = (cate_queue, db_cate, db_item)) for _ in range(5)]
  for item_t in item_th:
    item_t.start()
  cate_th.join()

希望本文所述对大家的Python程序设计有所帮助。