新增nginx绑定

services/nginx_service.go

@@ -0,0 +1,3 @@
+package services
+
+type NginxService struct{}

services/site_service.go

@@ -0,0 +1,175 @@
+package services
+
+import (
+	"go-site-clone/libs"
+	"log"
+	"net/url"
+	"strings"
+
+	"github.com/PuerkitoBio/goquery"
+	"github.com/go-rod/rod"
+	"github.com/go-rod/rod/lib/proto"
+)
+
+type SiteService struct {
+	BaseDomain string
+}
+
+type RequestParams struct {
+	Type proto.NetworkResourceType `json:"type"`
+	URL  string                    `json:"url"`
+}
+
+// 获取当前页面的所有本站资源 js、css、img、链接
+func (s SiteService) GetAllResources(rawURL string) ([]RequestParams, []string) {
+	var RequestParamsAll []RequestParams
+	var RouterAll []string
+	var PastHref map[string]struct{} = make(map[string]struct{})
+	var ActiveHref map[string]struct{} = make(map[string]struct{})
+	// 解析域名
+	parsed, err := url.Parse(rawURL)
+	if err != nil {
+		log.Println(err)
+	}
+
+	var Browser libs.Browser
+	host := parsed.Hostname()
+	if parsed.Port() != "" {
+		host += parsed.Port()
+	}
+	obj := Browser.GetBrowser(host, &libs.Fingerprint{})
+	s.BaseDomain = host
+	ActiveHref["https://"+host] = struct{}{}
+	// 进入主站
+	// browser := obj.Browser
+	page := obj.Page
+	s.LoopGet(page, PastHref, ActiveHref, &RequestParamsAll)
+	for k := range PastHref {
+		RouterAll = append(RouterAll, k)
+	}
+	list := deduplicationRequest(RequestParamsAll)
+	// 资源去重
+	return list, RouterAll
+}
+
+// 循环获取当前的资源及a标签中的链接
+func (s SiteService) LoopGet(page *rod.Page, pastHref map[string]struct{}, activeHref map[string]struct{}, requestParamsAll *[]RequestParams) {
+	// 取出
+	for len(activeHref) > 0 {
+		var url string
+		for k := range activeHref {
+			url = k
+			pastHref[k] = struct{}{}
+			delete(activeHref, k)
+			break
+		}
+		if url == "" {
+			return
+		}
+		log.Println("当前页面：", url)
+		page.MustNavigate(url)
+		// 提取当前页面的所有资源
+		rs := getApiResponse(page)
+		// 将当前资源加到全局资源中
+		*requestParamsAll = append(*requestParamsAll, rs...)
+		// 获取页面 HTML 提取当前页面的所有a标签
+		html, err := page.HTML()
+		if err != nil {
+			panic(err)
+		}
+		aList := s.GetSrcByHtml(html)
+		for _, v := range aList {
+			if _, seen := pastHref[v]; !seen {
+				activeHref[v] = struct{}{}
+			}
+		}
+	}
+
+}
+
+func getApiResponse(page *rod.Page) []RequestParams {
+	// 定义一个通道接收消息
+	var RequestParamsList []RequestParams
+	wait := page.EachEvent(func(e *proto.PageLoadEventFired) {
+		RequestParamsList = []RequestParams{}
+	}, func(e *proto.NetworkResponseReceived) {
+		RequestParamsList = append(RequestParamsList, RequestParams{
+			Type: e.Type,
+			URL:  e.Response.URL,
+		})
+	})
+	go wait()
+	page.MustWaitLoad()
+	return RequestParamsList
+}
+
+func (s SiteService) GetSrcByHtml(html string) []string {
+	urlList := []string{}
+	seen := make(map[string]struct{}) // 用 map 去重
+
+	doc, err := goquery.NewDocumentFromReader(strings.NewReader(html))
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	doc.Find("a").Each(func(i int, sel *goquery.Selection) {
+		href, exists := sel.Attr("href")
+		if !exists || href == "" {
+			return
+		}
+
+		// 过滤掉锚点和伪链接
+		if strings.HasPrefix(href, "#") || strings.HasPrefix(href, "javascript:") || href == "/" {
+			return
+		}
+
+		// 解析 URL
+		u, err := url.Parse(href)
+		if err != nil {
+			return
+		}
+
+		// 相对路径 → 拼接为绝对路径
+		if !u.IsAbs() {
+			if strings.HasPrefix(href, "/") {
+				href = "https://" + s.BaseDomain + u.Path
+			} else if strings.HasPrefix(href, "./") {
+				result := strings.TrimPrefix(href, ".")
+				href = "https://" + s.BaseDomain + result
+			} else {
+				href = "https://" + s.BaseDomain + "/" + u.Path
+			}
+
+		} else {
+			// 只保留站内链接
+			if !strings.Contains(u.Host, s.BaseDomain) {
+				return
+			}
+		}
+
+		// 去重
+		if _, ok := seen[href]; !ok {
+			seen[href] = struct{}{}
+			urlList = append(urlList, href)
+		}
+	})
+
+	return urlList
+}
+
+// 去重资源
+func deduplicationRequest(list []RequestParams) []RequestParams {
+	var newList []RequestParams
+	for _, v := range list {
+		is := true
+		for _, l := range newList {
+			if l.URL == v.URL && l.Type == v.Type {
+				is = false
+			}
+		}
+		if is {
+			newList = append(newList, v)
+		}
+	}
+	return newList
+}