golangcolly爬取第一ppt完整源码和程序（已编译好放在github)

git clone https://github.com/xhaoxiong/fetch_ppt
go build
./FetchPPT FetchPPT.exe

/*** &＃64;Author xiaoxiao* &＃64;Description CREATE FILE collector* &＃64;Date 2020/10/10 10:29 上午**/
package collectorimport ("FetchPPT/util""bytes""fmt""github.com/gocolly/colly/v2""io""log""os""path""strconv""strings""sync"
)const (originUrl &＃61; "http://www.di1ppt.com"downloadOriginUrl &＃61; "http://www.di1ppt.com/e/DownSys"
)type SeedConfig struct {DownloadFile DownloadFileGetNavCollector func(file DownloadFile)
}type DownloadFile struct {Filename stringUrl stringDirectory stringOriginDirectory string
}type CrawCollector struct {DownloadFile DownloadFileWg *sync.WaitGroup
}var filterMap &＃61; map[string]bool{"/office/": true,"/ziti/": true,"http://www.10103.com": true,
}var page_list &＃61; make(map[string]bool)func Run() {SeedConfig :&＃61; SeedConfig{DownloadFile: DownloadFile{Filename: "",Url: "",Directory: "",},GetNavCollector: GetNavCollector,}SeedConfig.Run()
}func (s *SeedConfig) Run() {s.GetNavCollector(s.DownloadFile)
}var Wg &＃61; &sync.WaitGroup{}//获取导航页面
func GetNavCollector(downloadFile DownloadFile) {c :&＃61; colly.NewCollector()c.OnHTML("#navMenu li", func(element *colly.HTMLElement) {cc :&＃61; &CrawCollector{DownloadFile: downloadFile,}seedUrl :&＃61; element.ChildAttr("a", "href")if !filterMap[seedUrl] {directory :&＃61; element.ChildText("a>span")dir :&＃61; path.Join(downloadFile.OriginDirectory, directory)if !util.Exists(dir) {os.MkdirAll(dir, 0777)}cc.DownloadFile.Directory &＃61; dirWg.Add(1)cc.GetDetailCollector(seedUrl)}})c.OnScraped(func(response *colly.Response) {fmt.Println("完成全部抓取")})c.Visit(originUrl)Wg.Wait()
}//获取导航对应首页N页列表
func (cc *CrawCollector) GetDetailCollector(seedUrl string) {c :&＃61; colly.NewCollector()c.OnHTML(".dlbox .clearfix .pages", func(element *colly.HTMLElement) {lis :&＃61; element.DOM.Find("li")pageUrl, _ :&＃61; lis.Last().Find("a").Attr("href")split :&＃61; strings.Split(pageUrl, "_")ii, _ :&＃61; strconv.Atoi(strings.Split(split[1], ".")[0])for i :&＃61; 1; i }//获取每页对应的详情页
func (cc *CrawCollector) GetPageDetailCollector(seedUrl2 string) {c :&＃61; colly.NewCollector()c.OnHTML(".dlbox .tplist li>a", func(element *colly.HTMLElement) {detailUrl :&＃61; element.Attr("href")Wg.Add(1)go cc.GetDownloadUrlCollector(detailUrl)})c.Visit(originUrl &＃43; seedUrl2)
}//获取下载页面
func (cc *CrawCollector) GetDownloadUrlCollector(detailUrl string) {c :&＃61; colly.NewCollector()c.OnHTML(".downurllist li>a", func(element *colly.HTMLElement) {if element.Index &＃61;&＃61; 0 {downloadlUrl :&＃61; element.Attr("href")cc.GetDownloadUrlDetailCollector(downloadlUrl)}})c.Visit(originUrl &＃43; detailUrl)
}//获取验证码下载页面
func (cc *CrawCollector) GetDownloadUrlDetailCollector(downLoadDetailUrl string) {c :&＃61; colly.NewCollector()c.OnHTML("tbody td>a", func(element *colly.HTMLElement) {downloadUrl :&＃61; element.Attr("href")downloadUrl &＃61; strings.Replace(downloadUrl, "..", "", -1)cc.DownloadFile.Url &＃61; downloadOriginUrl &＃43; downloadUrlWg.Add(1)go cc.FetchPPT(downloadOriginUrl &＃43; downloadUrl)})c.Visit(originUrl &＃43; downLoadDetailUrl)
}//获取ppt详情下载页面
func (cc *CrawCollector) FetchPPT(dowloadUrl string) {c :&＃61; colly.NewCollector()defer Wg.Done()c.OnResponse(func(response *colly.Response) {filename :&＃61; response.FileName()filepath :&＃61; path.Join(cc.DownloadFile.Directory, filename)if _, err :&＃61; os.Stat(filepath); err &＃61;&＃61; nil {log.Println("文件已存在:", filename)return}output, err :&＃61; os.Create(filepath)defer output.Close()if err !&＃61; nil {log.Println("创建失败: ", err)}_, err &＃61; io.Copy(output, bytes.NewReader(response.Body))if err !&＃61; nil {log.Println("写入失败 ", err)}log.Printf("下载文件 %s/%s", cc.DownloadFile.Directory, filename)})c.Visit(dowloadUrl)
}