Browse Source

fix:删除无用代码

master
燕鹏 3 years ago
parent
commit
b2cffc66e2
  1. 3
      spider/spider.go

3
spider/spider.go

@ -43,10 +43,10 @@ func SpiderAiprose() {
/// csdn爬取 https://blog.csdn.net/bitree1?type=blog /// csdn爬取 https://blog.csdn.net/bitree1?type=blog
func SpiderBitree() { func SpiderBitree() {
c := colly.NewCollector(colly.AllowedDomains("blog.csdn.net"), colly.Async(false)) c := colly.NewCollector(colly.AllowedDomains("blog.csdn.net"), colly.Async(false))
c.Async = false
c.UserAgent = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.75 Safari/537.36" c.UserAgent = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.75 Safari/537.36"
c.Limit(&colly.LimitRule{ c.Limit(&colly.LimitRule{
Delay: 10000 * time.Minute, Delay: 10000 * time.Minute,
//RandomDelay: 2 * time.Minute,
Parallelism: 1, Parallelism: 1,
}) })
c2 := c.Clone() c2 := c.Clone()
@ -62,7 +62,6 @@ func SpiderBitree() {
c2.Request("GET", e.Attr("href"), nil, nil, nil) c2.Request("GET", e.Attr("href"), nil, nil, nil)
}) })
c2.OnHTML(".blog-content-box", func(e *colly.HTMLElement) { c2.OnHTML(".blog-content-box", func(e *colly.HTMLElement) {
//fmt.Println("获取到文章")
title := strings.TrimSpace(e.DOM.Find(".title-article").Eq(0).Text()) title := strings.TrimSpace(e.DOM.Find(".title-article").Eq(0).Text())
fmt.Println(title) fmt.Println(title)
}) })

Loading…
Cancel
Save