From b2cffc66e2ef306f3bea5acb25bb35c23627d7e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=87=95=E9=B9=8F?= Date: Tue, 12 Oct 2021 14:28:23 +0800 Subject: [PATCH] =?UTF-8?q?fix:=E5=88=A0=E9=99=A4=E6=97=A0=E7=94=A8?= =?UTF-8?q?=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- spider/spider.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/spider/spider.go b/spider/spider.go index 3a6a3a0..c3c5e7f 100644 --- a/spider/spider.go +++ b/spider/spider.go @@ -43,10 +43,10 @@ func SpiderAiprose() { /// csdn爬取 https://blog.csdn.net/bitree1?type=blog func SpiderBitree() { c := colly.NewCollector(colly.AllowedDomains("blog.csdn.net"), colly.Async(false)) + c.Async = false c.UserAgent = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.75 Safari/537.36" c.Limit(&colly.LimitRule{ - Delay: 10000 * time.Minute, - //RandomDelay: 2 * time.Minute, + Delay: 10000 * time.Minute, Parallelism: 1, }) c2 := c.Clone() @@ -62,7 +62,6 @@ func SpiderBitree() { c2.Request("GET", e.Attr("href"), nil, nil, nil) }) c2.OnHTML(".blog-content-box", func(e *colly.HTMLElement) { - //fmt.Println("获取到文章") title := strings.TrimSpace(e.DOM.Find(".title-article").Eq(0).Text()) fmt.Println(title) })