goquery 爬虫实践

2025-02-14 技术教程

/*获取URL范例*//*Find 查找获取当前匹配的每个元素的后代Eq 选择第几个Attr 获取对应的标签属性AttrOr 获取对应的标签属性。这个可以设置第二个参数。获取的默认值如果获取不到默认调用对应默认值Each 遍历每一个元素Text 获取当前对应的文本Html 获取当前对象的标签AddClass 添加 class 不过用来抓取有点鸡肋不知道为何要写这个Children 返回所有子元素Filter 过滤标签元素Prev 获取上一个元素Next 获取下一个元素*/package mainimport ( "fmt" "log" "os" "regexp" "strconv" "github.com/PuerkitoBio/goquery")func getdata(ins int, ch chan int) { url := "" if ins == 1 { url = "https://colobu.com/categories/Go" } else { url = "https://colobu.com/categories/Go/page/" + strconv.Itoa(ins) + "/" } doc, err := goquery.NewDocument(url) if err != nil { log.Fatal(err) } // <a class="article-title" href="/2019/06/01/packet-capture-injection-and-analysis-gopacket/">[译]利用 gopackage 进行包的捕获、注入和分析</a> doc.Find(".article-title").Each(func(i int, s *goquery.Selection) { a, _ := s.Attr("href") text := s.Text() a = "https://colobu.com" + a //htmls, _ := s.Html() fmt.Println("") fmt.Println("") fmt.Println(" 地址：" + a) fmt.Println(" 标题：" + text) /*text = strings.ReplaceAll(text, ":", " ") text = strings.ReplaceAll(text, "/", " ") text = strings.ReplaceAll(text, "\\", " ") text = strings.ReplaceAll(text, "?", " ") text = strings.ReplaceAll(text, "*", " ") */ reg := regexp.MustCompile(`:|\?|/|\*|<|>|"`) tilte := reg.ReplaceAllString(text, " ") docm, err := goquery.NewDocument(a) if err != nil { log.Fatal(err) } sstext := "" docm.Find(".article-entry").Each(func(ii int, ss *goquery.Selection) { sstext = ss.Text() }) //fmt.Println(" 正文：" + sstext) file, _ := os.OpenFile("./爬虫/第"+strconv.Itoa(ins)+"页 "+strconv.Itoa(i+1)+"篇 "+tilte+"页爬虫.txt", os.O_RDWR|os.O_TRUNC|os.O_CREATE, 0666) defer file.Close() file.Write([]byte(text + "\n正文:\n" + sstext + "\n\n\n")) fmt.Println(" ---------------------------------------------------------------------------- ") }) ch <- ins}func Doing(s, e int) { ch := make(chan int) for i := s; i <= e; i++ { go getdata(i, ch) } for i := s; i <= e; i++ { n := <-ch fmt.Printf("第%d页爬取完毕\n", n) }}func main() { var start, end int fmt.Println("输入起始页") fmt.Scan(&start) fmt.Println("输入终止页") fmt.Scan(&end) Doing(start, end)}

package mainimport ( "fmt" "io" "net/http" "os" "strconv")func HttpGet(url string) (res string, err error) { fmt.Println(url) resp, err1 := http.Get(url) if err1 != nil { err = err1 //fmt.Println(err) return } //fmt.Println(resp.Body) defer resp.Body.Close() buf := make([]byte, 4096) for { n, err2 := resp.Body.Read(buf) if n == 0 { fmt.Println("读取完毕") break } if err2 != nil && err2 != io.EOF { //fmt.Println(err2) err = err2 return } res += string(buf[:n]) } return}func working(start, end int) { fmt.Printf("正在爬取%d页面到%d页", start, end) for i := start; i <= end; i++ { url := "http://tieba.baidu.com/f?kw=%E5%88%AB%E5%85%8B&ie=utf-8&pn=" + strconv.Itoa((i-1)*50) //resp, err := http.Get(url) result, err := HttpGet(url) if err != nil { fmt.Println(err) continue } fmt.Println(result) file, err := os.Create("第" + strconv.Itoa(i) + "页面.html") if err != nil { fmt.Println(err) } file.WriteString(result) file.Close() }}func main() { var start, end int fmt.Println("请输入爬取的起始页（》=1）：") fmt.Scan(&start) fmt.Println("请输入爬取的结束页（》=start）：") fmt.Scan(&end) working(start, end)}

package mainimport ( "fmt" "io" "net/http" "os" "strconv")func HttpGet(url string) (res string, err error) { fmt.Println(url) resp, err1 := http.Get(url) if err1 != nil { err = err1 //fmt.Println(err) return } //fmt.Println(resp.Body) defer resp.Body.Close() buf := make([]byte, 4096) for { n, err2 := resp.Body.Read(buf) if n == 0 { //fmt.Println("读取完毕") break } if err2 != nil && err2 != io.EOF { //fmt.Println(err2) err = err2 return } res += string(buf[:n]) } return}func getdata(i int, ch chan int) { url := "http://tieba.baidu.com/f?kw=%E5%88%AB%E5%85%8B&ie=utf-8&pn=" + strconv.Itoa((i-1)*50) //resp, err := http.Get(url) fmt.Println("第" + strconv.Itoa(i) + "页面.html") result, err := HttpGet(url) if err != nil { fmt.Println(err) //continue } //fmt.Println(result) file, err := os.Create("第" + strconv.Itoa(i) + "页面.html") if err != nil { fmt.Println(err) } file.WriteString(result) file.Close() ch <- i}func working(s, e int) { ch := make(chan int) for i := s; i <= e; i++ { go getdata(i, ch) } for i := s; i <= e; i++ { n := <-ch fmt.Printf("第%d页爬取完毕\n", n) }}func main() { var start, end int fmt.Println("请输入爬取的起始页（》=1）：") fmt.Scan(&start) fmt.Println("请输入爬取的结束页（》=start）：") fmt.Scan(&end) working(start, end)}