第十三章go实现分布式网络爬虫单机版爬虫

package main

import (
    "fmt"
    "io/ioutil"
    "net/http"
    "regexp"
)

func main() {
    // 第一步, 通过url抓取页面
    resp, err := http.Get("http://www.zhenai.com/zhenghun")
    if err != nil {
        panic(err)
    }
    defer resp.Body.Close()

    if resp.StatusCode != http.StatusOK {
        return
    }

    // 读取出来body的所有内容
    all, err := ioutil.ReadAll(resp.Body)
    if err != nil {
        panic(err)
    }
    //fmt.Printf("%s\n", all)
    printCityList(all)
}

package main

import (
    "fmt"
    "io/ioutil"
    "net/http"
    "regexp"
)

func main() {
    // 第一步, 通过url抓取页面
    resp, err := http.Get("http://www.zhenai.com/zhenghun")
    if err != nil {
        panic(err)
    }
    defer resp.Body.Close()

    if resp.StatusCode != http.StatusOK {
        return
    }

    // 读取出来body的所有内容
    all, err := ioutil.ReadAll(resp.Body)
    if err != nil {
        panic(err)
    }
    //fmt.Printf("%s\n", all)
    printCityList(all)
}

/**
 * 正则表达式提取城市名称和跳转的url
 */
func printCityList(content []byte) {
    re := regexp.MustCompile(`"(http://www.zhenai.com/zhenghun/[a-z1-9]+)" data-v-5e16505f>([^<]+)`)
    all := re.FindAllSubmatch(content, -1)
    for _, line := range all {
        fmt.Printf("city: %s, url: %s\n", line[2], line[1])

    }
}

package fetcher

import (
    "fmt"
    "io/ioutil"
    "net/http"
)
// 抓取器
func Fetch(url string) ([]byte, error) {

    // 第一步, 通过url抓取页面
    client := http.Client{}
    request, err := http.NewRequest("GET", url, nil)
    request.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.181 Safari/537.36")
    resp, err := client.Do(request)
    //resp, err := http.Get(url)
    if err != nil {
        return nil, fmt.Errorf("http get error :%s", err)
    }
    defer resp.Body.Close()

    if resp.StatusCode != http.StatusOK {
        return nil, fmt.Errorf("http get error errCode:%d", http.StatusOK)
    }

    // 读取出来body的所有内容
    return ioutil.ReadAll(resp.Body)
}

package parser

import (
    "aaa/crawler/zhenai/engine"
    "regexp"
)

const cityListRegexp  = `"(http://www.zhenai.com/zhenghun/[a-z1-9]+)"[^>]*>([^<]+)`
func ParseCityList(content []byte) (engine.ParseResult) {
    re := regexp.MustCompile(cityListRegexp)
    all := re.FindAllSubmatch(content, -1)
    pr := engine.ParseResult{}
    count := 1
    for _, line := range all {
        req := engine.Request{
            Url:string(line[1]), 
            ParseFun: ParseCity,
        }
        pr.Req = append(pr.Req, req)

        pr.Items = append(pr.Items, "City: " + string(line[2]))

        count --
        if count <=0 {
            break
        }
    }
    return pr
}

package engine

type Request struct {
    Url string
    ParseFun func(content []byte) ParseResult
}

type ParseResult struct {
    Req []Request
    Items []interface{}
}

func NilParse(content []byte) ParseResult{
    return ParseResult{}
}

package engine

import (
    "aaa/crawler/fetcher"
    "fmt"
    "github.com/astaxie/beego/logs"
)

func Run(seeds ...Request) {

    var que []Request

    for _, seed := range seeds {
        que = append(que, seed)
    }

    for len(que) > 0 {
        cur := que[0]
        que = que[1:]

        logs.Info("fetch url:", cur.Url)
        cont, e := fetcher.Fetch(cur.Url)
        if e != nil {
            logs.Info("解析页面异常 url:", cur.Url)
            continue
        }

        resultParse := cur.ParseFun(cont)
        que = append(que, resultParse.Req...)

        for _, item := range resultParse.Items {
            fmt.Printf("内容项: %s \n", item)
        }
    }
}

package main

import (
    "aaa/crawler/zhenai/engine"
    "aaa/crawler/zhenai/parser"
)

func main() {
    req := engine.Request{
        Url:"http://www.zhenai.com/zhenghun", 
        ParseFun: parser.ParseCityList,
    }
    engine.Run(req)

}

package parser

import (
    "aaa/crawler/zhenai/engine"
    "regexp"
)

const cityRe = `"(http://album.zhenai.com/u/[0-9]+)"[^>]*>([^<]+)`
func ParseCity(content []byte) engine.ParseResult{

    cityRegexp:= regexp.MustCompile(cityRe)
    subs := cityRegexp.FindAllSubmatch(content, -1)
    pr := engine.ParseResult{}
    for _, sub := range subs {
        name := string(sub[2])
        // 获取用户的详细地址
        re := engine.Request{
            Url:string(sub[1]),
            // 注意, 这里定义了一个函数来传递, 这样可以吧name也传递过去
            ParseFun: func(content []byte) engine.ParseResult {
                return ParseUser(content, name)
            },
        }
        pr.Req = append(pr.Req, re)

        pr.Items = append(pr.Items, "Name: " + string(sub[2]))
    }

    return pr
}

package parser

import (
    "aaa/crawler/zhenai/engine"
    "aaa/crawler/zhenai/model"
    "regexp"
    "strconv"
    "strings"
)

// 个人基本信息
const userRegexp = `class]*class="m-btn purple"[^>]*>([^<]+)