diff --git a/spider/spider.go b/spider/spider.go index 3a6a3a0..c3c5e7f 100644 --- a/spider/spider.go +++ b/spider/spider.go @@ -43,10 +43,10 @@ func SpiderAiprose() { /// csdn爬取 https://blog.csdn.net/bitree1?type=blog func SpiderBitree() { c := colly.NewCollector(colly.AllowedDomains("blog.csdn.net"), colly.Async(false)) + c.Async = false c.UserAgent = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.75 Safari/537.36" c.Limit(&colly.LimitRule{ - Delay: 10000 * time.Minute, - //RandomDelay: 2 * time.Minute, + Delay: 10000 * time.Minute, Parallelism: 1, }) c2 := c.Clone() @@ -62,7 +62,6 @@ func SpiderBitree() { c2.Request("GET", e.Attr("href"), nil, nil, nil) }) c2.OnHTML(".blog-content-box", func(e *colly.HTMLElement) { - //fmt.Println("获取到文章") title := strings.TrimSpace(e.DOM.Find(".title-article").Eq(0).Text()) fmt.Println(title) })