go colly 爬虫博客到wordpress
https://www.aispider.cc
Non puoi selezionare più di 25 argomenti
Gli argomenti devono iniziare con una lettera o un numero, possono includere trattini ('-') e possono essere lunghi fino a 35 caratteri.
36 righe
1.2 KiB
36 righe
1.2 KiB
package spider
|
|
|
|
import (
|
|
"fmt"
|
|
"github.com/antchfx/htmlquery"
|
|
"github.com/gocolly/colly/v2"
|
|
"strings"
|
|
)
|
|
|
|
func SpiderAiprose() {
|
|
c := colly.NewCollector(colly.AllowedDomains("www.aiprose.com"), colly.Async(true))
|
|
c.OnHTML(".home-content-title a[href]", func(e *colly.HTMLElement) {
|
|
e.Request.Visit(e.Attr("href"))
|
|
})
|
|
c.OnHTML(".pagination .next a[href]", func(e *colly.HTMLElement) {
|
|
e.Request.Visit(e.Attr("href"))
|
|
})
|
|
c.OnHTML(".blog-root", func(e *colly.HTMLElement) {
|
|
title := strings.TrimSpace(e.DOM.Find(".blog-title").Eq(0).Text())
|
|
//author:=strings.TrimSpace(e.DOM.Find(".author-info a").Eq(0).Text())
|
|
time := strings.TrimSpace(e.DOM.Find(".author-info span").Eq(0).Text())
|
|
content := strings.TrimSpace(e.DOM.Find(".blog-detaile").Eq(0).Text())
|
|
nodes := e.DOM.Find(".blog-label label").Nodes
|
|
var tags []string
|
|
for _, node := range nodes {
|
|
tags = append(tags, htmlquery.InnerText(node))
|
|
}
|
|
saveBlog(title, content, time, tags)
|
|
})
|
|
c.OnRequest(func(r *colly.Request) {
|
|
fmt.Println("Visiting", r.URL)
|
|
})
|
|
c.UserAgent = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.75 Safari/537.36"
|
|
c.Visit("https://www.aiprose.com/blogs")
|
|
c.Wait()
|
|
}
|
|
|