蚂蜂窝爬虫

function getArticleList(pageNum) {
request({ url: "http://www.mafengwo.cn/ajax/ajax_article.php?start&＃61;" &＃43; pageNum, headers: { &＃39;User-Agent&＃39;: &＃39;Mozilla/5.0&＃39; } }, function(error, response, data) { var res &＃61; data.match(/i\\\/\d{7}/g); for (var i &＃61; 0; i <12; i&＃43;&＃43;) { articlesUrl[i] &＃61; res[i * 3].substr(3, 7); }; async.each(articlesUrl, getArticle, function(err) { console.log(&＃39;err: &＃39; &＃43; err); }); }); }

function getArticle(urlNumber) {
request({ url: "http://www.mafengwo.cn/i/" &＃43; urlNumber &＃43; ".html", headers: { &＃39;User-Agent&＃39;: &＃39;Mozilla/5.0&＃39; } }, function(error, response, data) { //处理数据 }); };

var title, content, creator, created;
/*获取标题*/
title &＃61; data.match(/\s*.&＃43;\s*<\/h1>/).toString().replace(/\s*/g, "").replace(/$/g, "").replace(/\//g, "|").match(/>.&＃43;title &＃61; title.substring(1, title.length - 1);

/*如果有背景音乐就获取背景音乐*/
if (data.indexOf("music_url") &＃39;music_auto_play&＃39;)) { mp3url &＃61; data.substring(data.indexOf("music_url"), data.indexOf(&＃39;music_auto_play&＃39;)); } else { mp3url &＃61; data.substring(data.indexOf("music_url"), data.indexOf(&＃39;is_new_note&＃39;)); }; mp3url &＃61; mp3url.match(/http.&＃43;\.mp3/); if (mp3url) { mp3url &＃61; mp3url.toString(); content &＃61; &＃39;&＃39;; };

/*获取文章内容&＃xff0c;发现有两种类型&＃xff0c;分别适配*/
if (data.indexOf(&＃39;a_con_text cont&＃39;) !&＃61; -1) { content &＃43;&＃61; data.substring(data.indexOf("a_con_text cont") &＃43; 296, data.indexOf(&＃39;integral&＃39;) - 12); } else { content &＃43;&＃61; data.substring(data.indexOf("ginfo_kw_hotel") &＃43; 16, data.indexOf(&＃39;vc_total&＃39;) - 19); };

/*移除它给图片定义的父标签宽度以便响应式*/
content &＃61; content.replace(/width:\d*px/g, "");
/*把文中˚∫圖片作為在列表中顯示時的圖片*/ /*有的第一張圖片是表情.....處理一下..*/ imageUrl &＃61; data.match(/http.*\.(jpeg|png|jpg)"/).toString(); imageUrl &＃61; imageUrl.substring(0, imageUrl.indexOf(&＃39;"&＃39;));

fs.writeFile("html/" &＃43; title &＃43; ".html", content, function(e) { if (e) throw e; console.log(title); });

MongoClient.connect(&＃39;mongodb://localhost:27017/mean&＃39;, function(err, db) {
assert.equal(null, err); insertArticle(db, title, content, creator, mp3url, imageUrl, created, function() { db.close(); }); });

var ArticleSchema &＃61; new Schema({
created: {
type: Date, default: Date.now }, title: { type: String, default: &＃39;&＃39;, trim: true, required: &＃39;Title cannot be blank&＃39; }, content: { type: String, default: &＃39;&＃39;, trim: true }, mp3url:{ type:String }, imageUrl:{ type:String }, creator: { type: String, default: &＃39;golmic&＃39;, } });

var UserSchema &＃61; new Schema({
firstName: String,
lastName: String, email: { type: String, // Validate the email format match: [/.&＃43;\&＃64;.&＃43;\..&＃43;/, "Please fill a valid email address"] }, username: { type: String, // Set a unique &＃39;username&＃39; index unique: true, // Validate &＃39;username&＃39; value existance required: &＃39;Username is required&＃39;, // Trim the &＃39;username&＃39; field trim: true }, password: { type: String, // Validate the &＃39;password&＃39; value length validate: [ function(password) { return password && password.length > 6; }, &＃39;Password should be longer&＃39; ] }, salt: { type: String }, provider: { type: String, // Validate &＃39;provider&＃39; value existance required: &＃39;Provider is required&＃39; }, providerId: String, providerData: {}, created: { type: Date, // Create a default &＃39;created&＃39; value default: Date.now } });

exports.list &＃61; function(req, res) {
Article.find().sort(&＃39;-created&＃39;).exec(function(err, articles) { if (err) { return res.status(400).send({ message: getErrorMessage(err) }); } else { for(var i in articles){ articles[i].content&＃61;&＃39;&＃39;; }; res.json(articles); } }); }; exports.read &＃61; function(req, res) { res.json(req.article); }; exports.update &＃61; function(req, res) { var article &＃61; req.article; article.title &＃61; req.body.title; article.content &＃61; req.body.content; article.save(function(err) { if (err) { return res.status(400).send({ message: getErrorMessage(err) }); } else { res.json(article); } }); }; exports.delete &＃61; function(req, res) { var article &＃61; req.article; article.remove(function(err) { if (err) { return res.status(400).send({ message: getErrorMessage(err) }); } else { res.json(article); } }); };

module.exports &＃61; function(app) {
app.route(&＃39;/api/articles&＃39;) .get(articles.list) .post(users.requiresLogin, articles.create); app.route(&＃39;/api/articles/:articleId&＃39;) .get(articles.read) .put(users.requiresLogin, articles.hasAuthorization, articles.update) .delete(users.requiresLogin, articles.hasAuthorization, articles.delete); app.param(&＃39;articleId&＃39;, articles.articleByID); };

angular.module(&＃39;articles&＃39;).config([&＃39;$routeProvider&＃39;,
function($routeProvider) { $routeProvider. when(&＃39;/&＃39;, { templateUrl: &＃39;articles/views/list-articles.client.view.html&＃39; }). when(&＃39;/articles/create&＃39;, { templateUrl: &＃39;articles/views/create-article.client.view.html&＃39; }). when(&＃39;/articles/:articleId&＃39;, { templateUrl: &＃39;articles/views/view-article.client.view.html&＃39; }). when(&＃39;/articles/:articleId/edit&＃39;, { templateUrl: &＃39;articles/views/edit-article.client.view.html&＃39; }); } ]);

angular.module(&＃39;articles&＃39;).factory(&＃39;Articles&＃39;, [&＃39;$resource&＃39;, function($resource) { // Use the &＃39;$resource&＃39; service to return an article &＃39;$resource&＃39; object return $resource(&＃39;api/articles/:articleId&＃39;, { articleId: &＃39;&＃64;_id&＃39; }, { update: { method: &＃39;PUT&＃39; } }); }]);