package services import ( "go-site-clone/libs" "log" "net/url" "strings" "github.com/PuerkitoBio/goquery" "github.com/go-rod/rod" "github.com/go-rod/rod/lib/proto" ) type SiteService struct { BaseDomain string } type RequestParams struct { Type proto.NetworkResourceType `json:"type"` URL string `json:"url"` } // 获取当前页面的所有本站资源 js、css、img、链接 func (s SiteService) GetAllResources(rawURL string) ([]RequestParams, []string) { var RequestParamsAll []RequestParams var RouterAll []string var PastHref map[string]struct{} = make(map[string]struct{}) var ActiveHref map[string]struct{} = make(map[string]struct{}) // 解析域名 parsed, err := url.Parse(rawURL) if err != nil { log.Println(err) } var Browser libs.Browser host := parsed.Hostname() if parsed.Port() != "" { host += parsed.Port() } obj := Browser.GetBrowser(host, &libs.Fingerprint{}) s.BaseDomain = host ActiveHref["https://"+host] = struct{}{} // 进入主站 // browser := obj.Browser page := obj.Page s.LoopGet(page, PastHref, ActiveHref, &RequestParamsAll) for k := range PastHref { RouterAll = append(RouterAll, k) } list := deduplicationRequest(RequestParamsAll) // 资源去重 return list, RouterAll } // 循环获取当前的资源及a标签中的链接 func (s SiteService) LoopGet(page *rod.Page, pastHref map[string]struct{}, activeHref map[string]struct{}, requestParamsAll *[]RequestParams) { // 取出 for len(activeHref) > 0 { var url string for k := range activeHref { url = k pastHref[k] = struct{}{} delete(activeHref, k) break } if url == "" { return } log.Println("当前页面:", url) page.MustNavigate(url) // 提取当前页面的所有资源 rs := getApiResponse(page) // 将当前资源加到全局资源中 *requestParamsAll = append(*requestParamsAll, rs...) // 获取页面 HTML 提取当前页面的所有a标签 html, err := page.HTML() if err != nil { panic(err) } aList := s.GetSrcByHtml(html) for _, v := range aList { if _, seen := pastHref[v]; !seen { activeHref[v] = struct{}{} } } } } func getApiResponse(page *rod.Page) []RequestParams { // 定义一个通道接收消息 var RequestParamsList []RequestParams wait := page.EachEvent(func(e *proto.PageLoadEventFired) { RequestParamsList = []RequestParams{} }, func(e *proto.NetworkResponseReceived) { RequestParamsList = append(RequestParamsList, RequestParams{ Type: e.Type, URL: e.Response.URL, }) }) go wait() page.MustWaitLoad() return RequestParamsList } func (s SiteService) GetSrcByHtml(html string) []string { urlList := []string{} seen := make(map[string]struct{}) // 用 map 去重 doc, err := goquery.NewDocumentFromReader(strings.NewReader(html)) if err != nil { log.Fatal(err) } doc.Find("a").Each(func(i int, sel *goquery.Selection) { href, exists := sel.Attr("href") if !exists || href == "" { return } // 过滤掉锚点和伪链接 if strings.HasPrefix(href, "#") || strings.HasPrefix(href, "javascript:") || href == "/" { return } // 解析 URL u, err := url.Parse(href) if err != nil { return } // 相对路径 → 拼接为绝对路径 if !u.IsAbs() { if strings.HasPrefix(href, "/") { href = "https://" + s.BaseDomain + u.Path } else if strings.HasPrefix(href, "./") { result := strings.TrimPrefix(href, ".") href = "https://" + s.BaseDomain + result } else { href = "https://" + s.BaseDomain + "/" + u.Path } } else { // 只保留站内链接 if !strings.Contains(u.Host, s.BaseDomain) { return } } // 去重 if _, ok := seen[href]; !ok { seen[href] = struct{}{} urlList = append(urlList, href) } }) return urlList } // 去重资源 func deduplicationRequest(list []RequestParams) []RequestParams { var newList []RequestParams for _, v := range list { is := true for _, l := range newList { if l.URL == v.URL && l.Type == v.Type { is = false } } if is { newList = append(newList, v) } } return newList }