|
|
|
@ -43,10 +43,10 @@ func SpiderAiprose() { |
|
|
|
/// csdn爬取 https://blog.csdn.net/bitree1?type=blog
|
|
|
|
func SpiderBitree() { |
|
|
|
c := colly.NewCollector(colly.AllowedDomains("blog.csdn.net"), colly.Async(false)) |
|
|
|
c.Async = false |
|
|
|
c.UserAgent = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.75 Safari/537.36" |
|
|
|
c.Limit(&colly.LimitRule{ |
|
|
|
Delay: 10000 * time.Minute, |
|
|
|
//RandomDelay: 2 * time.Minute,
|
|
|
|
Delay: 10000 * time.Minute, |
|
|
|
Parallelism: 1, |
|
|
|
}) |
|
|
|
c2 := c.Clone() |
|
|
|
@ -62,7 +62,6 @@ func SpiderBitree() { |
|
|
|
c2.Request("GET", e.Attr("href"), nil, nil, nil) |
|
|
|
}) |
|
|
|
c2.OnHTML(".blog-content-box", func(e *colly.HTMLElement) { |
|
|
|
//fmt.Println("获取到文章")
|
|
|
|
title := strings.TrimSpace(e.DOM.Find(".title-article").Eq(0).Text()) |
|
|
|
fmt.Println(title) |
|
|
|
}) |
|
|
|
|
