import pymongo
from scrapy.conf import settings
import pymysql
class Mongodb_Pipeline(object):
def __init__(self):
self.client = pymongo.MongoClient()
self.db = self.client[settings.get('MONGODB_NAME')]
def process_item(self, item, spider):
if item:
coll = item['classification1']
self.db[coll].insert(dict(item))
return item
def close_spider(self):
self.client.close()