From 60f4995dc713f0ffc24e49cc8c5b371c73f0fdcd Mon Sep 17 00:00:00 2001 From: zyj <18107291228@163.com> Date: Tue, 21 Oct 2025 18:30:40 +0800 Subject: [PATCH] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E4=B8=8B=E8=BD=BD=E9=A1=B5?= =?UTF-8?q?=E9=9D=A2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app.go | 52 + config.yaml | 3 + frontend/app/pages/site/download.vue | 952 ++++++++++++++---- frontend/bindings/go-site-clone/app.js | 32 + frontend/bindings/go-site-clone/index.js | 2 + .../bindings/go-site-clone/services/index.js | 7 + .../bindings/go-site-clone/services/models.js | 75 ++ frontend/nuxt.config.ts | 2 +- go.mod | 2 +- go.sum | 13 +- main.go | 6 +- services/site_service.go | 11 +- 12 files changed, 953 insertions(+), 204 deletions(-) create mode 100644 app.go create mode 100644 frontend/bindings/go-site-clone/app.js create mode 100644 frontend/bindings/go-site-clone/services/index.js create mode 100644 frontend/bindings/go-site-clone/services/models.js diff --git a/app.go b/app.go new file mode 100644 index 0000000..3fe0eec --- /dev/null +++ b/app.go @@ -0,0 +1,52 @@ +package main + +import ( + "go-site-clone/services" + + "github.com/go-rod/rod/lib/proto" + "github.com/wailsapp/wails/v3/pkg/application" +) + +type App struct { + app *application.App +} + +var siteService services.SiteService + +func (a *App) GetResources(rawURL string) *services.ResourcesList { + RequestParamsAll, RouterAll := siteService.GetAllResources(rawURL) + // 再次去重 根据url链接去除重复项 + dataList := siteService.DeduplicationRequestByUrl(RequestParamsAll) + resources := &services.ResourcesList{ + Dom: RouterAll, + } + // 将页面及资源一起返回 + for _, v := range dataList { + if v.Type == proto.NetworkResourceTypeImage { + resources.Image = append(resources.Image, v.URL) + } + if v.Type == proto.NetworkResourceTypeStylesheet { + resources.CSS = append(resources.CSS, v.URL) + } + if v.Type == proto.NetworkResourceTypeScript { + resources.Script = append(resources.Script, v.URL) + } + } + + return resources +} + +func (a *App) DownloadSite() { + // 将页面及资源一起返回 + // for _, v := range dataList { + // u, _ := url.Parse(v.URL) + // if v.Type == proto.NetworkResourceTypeImage && u.Hostname() == parsed.Hostname() { + // var File utils.File + // File.Download(v.URL) + // } + // } + // for _, v := range RouterAll { + // var File utils.File + // File.HTMLDownload(v) + // } +} diff --git a/config.yaml b/config.yaml index bdf1a30..cabb2a0 100644 --- a/config.yaml +++ b/config.yaml @@ -11,3 +11,6 @@ stunUrl: "stun:106.12.33.188:3478" # api地址 apiUrl: "http://106.12.33.188:6996" + +# 网页文件保存目录 +siteFileDir: "www" diff --git a/frontend/app/pages/site/download.vue b/frontend/app/pages/site/download.vue index 2bfb420..5735ef3 100644 --- a/frontend/app/pages/site/download.vue +++ b/frontend/app/pages/site/download.vue @@ -1,288 +1,862 @@ - + +/* 加载状态 */ +.loading-more { + display: flex; + justify-content: center; + padding: 2rem; +} + \ No newline at end of file diff --git a/frontend/bindings/go-site-clone/app.js b/frontend/bindings/go-site-clone/app.js new file mode 100644 index 0000000..db4616c --- /dev/null +++ b/frontend/bindings/go-site-clone/app.js @@ -0,0 +1,32 @@ +// @ts-check +// Cynhyrchwyd y ffeil hon yn awtomatig. PEIDIWCH Â MODIWL +// This file is automatically generated. DO NOT EDIT + +// eslint-disable-next-line @typescript-eslint/ban-ts-comment +// @ts-ignore: Unused imports +import { Call as $Call, CancellablePromise as $CancellablePromise, Create as $Create } from "@wailsio/runtime"; + +// eslint-disable-next-line @typescript-eslint/ban-ts-comment +// @ts-ignore: Unused imports +import * as services$0 from "./services/models.js"; + +/** + * @returns {$CancellablePromise} + */ +export function DownloadSite() { + return $Call.ByID(2539977978); +} + +/** + * @param {string} rawURL + * @returns {$CancellablePromise} + */ +export function GetResources(rawURL) { + return $Call.ByID(2167352808, rawURL).then(/** @type {($result: any) => any} */(($result) => { + return $$createType1($result); + })); +} + +// Private type creation functions +const $$createType0 = services$0.ResourcesList.createFrom; +const $$createType1 = $Create.Nullable($$createType0); diff --git a/frontend/bindings/go-site-clone/index.js b/frontend/bindings/go-site-clone/index.js index fdf1ff4..bc8d4bc 100644 --- a/frontend/bindings/go-site-clone/index.js +++ b/frontend/bindings/go-site-clone/index.js @@ -2,7 +2,9 @@ // Cynhyrchwyd y ffeil hon yn awtomatig. PEIDIWCH Â MODIWL // This file is automatically generated. DO NOT EDIT +import * as App from "./app.js"; import * as GreetService from "./greetservice.js"; export { + App, GreetService }; diff --git a/frontend/bindings/go-site-clone/services/index.js b/frontend/bindings/go-site-clone/services/index.js new file mode 100644 index 0000000..a1efb15 --- /dev/null +++ b/frontend/bindings/go-site-clone/services/index.js @@ -0,0 +1,7 @@ +// @ts-check +// Cynhyrchwyd y ffeil hon yn awtomatig. PEIDIWCH Â MODIWL +// This file is automatically generated. DO NOT EDIT + +export { + ResourcesList +} from "./models.js"; diff --git a/frontend/bindings/go-site-clone/services/models.js b/frontend/bindings/go-site-clone/services/models.js new file mode 100644 index 0000000..4d489d4 --- /dev/null +++ b/frontend/bindings/go-site-clone/services/models.js @@ -0,0 +1,75 @@ +// @ts-check +// Cynhyrchwyd y ffeil hon yn awtomatig. PEIDIWCH Â MODIWL +// This file is automatically generated. DO NOT EDIT + +// eslint-disable-next-line @typescript-eslint/ban-ts-comment +// @ts-ignore: Unused imports +import { Create as $Create } from "@wailsio/runtime"; + +export class ResourcesList { + /** + * Creates a new ResourcesList instance. + * @param {Partial} [$$source = {}] - The source object to create the ResourcesList. + */ + constructor($$source = {}) { + if (!("script" in $$source)) { + /** + * @member + * @type {string[]} + */ + this["script"] = []; + } + if (!("css" in $$source)) { + /** + * @member + * @type {string[]} + */ + this["css"] = []; + } + if (!("image" in $$source)) { + /** + * @member + * @type {string[]} + */ + this["image"] = []; + } + if (!("dom" in $$source)) { + /** + * @member + * @type {string[]} + */ + this["dom"] = []; + } + + Object.assign(this, $$source); + } + + /** + * Creates a new ResourcesList instance from a string or object. + * @param {any} [$$source = {}] + * @returns {ResourcesList} + */ + static createFrom($$source = {}) { + const $$createField0_0 = $$createType0; + const $$createField1_0 = $$createType0; + const $$createField2_0 = $$createType0; + const $$createField3_0 = $$createType0; + let $$parsedSource = typeof $$source === 'string' ? JSON.parse($$source) : $$source; + if ("script" in $$parsedSource) { + $$parsedSource["script"] = $$createField0_0($$parsedSource["script"]); + } + if ("css" in $$parsedSource) { + $$parsedSource["css"] = $$createField1_0($$parsedSource["css"]); + } + if ("image" in $$parsedSource) { + $$parsedSource["image"] = $$createField2_0($$parsedSource["image"]); + } + if ("dom" in $$parsedSource) { + $$parsedSource["dom"] = $$createField3_0($$parsedSource["dom"]); + } + return new ResourcesList(/** @type {Partial} */($$parsedSource)); + } +} + +// Private type creation functions +const $$createType0 = $Create.Array($Create.Any); diff --git a/frontend/nuxt.config.ts b/frontend/nuxt.config.ts index c2167f0..4970a3c 100644 --- a/frontend/nuxt.config.ts +++ b/frontend/nuxt.config.ts @@ -20,5 +20,5 @@ export default defineNuxtConfig({ }, plugins: [ '~/plugins/ant-design-vue', - ], + ] }) diff --git a/go.mod b/go.mod index 278e4c0..1d3ed0e 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module go-site-clone go 1.24.0 require ( + github.com/PuerkitoBio/goquery v1.10.3 github.com/go-rod/rod v0.116.2 github.com/go-rod/stealth v0.4.9 github.com/wailsapp/wails/v3 v3.0.0-alpha.27 @@ -13,7 +14,6 @@ require ( dario.cat/mergo v1.0.1 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect github.com/ProtonMail/go-crypto v1.1.6 // indirect - github.com/PuerkitoBio/goquery v1.10.3 // indirect github.com/adrg/xdg v0.5.3 // indirect github.com/andybalholm/cascadia v1.3.3 // indirect github.com/bep/debounce v1.2.1 // indirect diff --git a/go.sum b/go.sum index 68bbde9..7e8274f 100644 --- a/go.sum +++ b/go.sum @@ -142,8 +142,7 @@ golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliY golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= -golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34= -golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc= +golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE= golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc= golang.org/x/exp v0.0.0-20250210185358-939b2ce775ac h1:l5+whBCLH3iH2ZNHYLbAe58bo7yrN4mVcnkHDYz5vvs= golang.org/x/exp v0.0.0-20250210185358-939b2ce775ac/go.mod h1:hH+7mtFmImwwcMvScyxUhjuVHR3HGaDPMn9rMSUUbxo= @@ -163,8 +162,6 @@ golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= -golang.org/x/net v0.37.0 h1:1zLorHbz+LYj7MQlSf1+2tPIIgibq2eL5xkrGk6f+2c= -golang.org/x/net v0.37.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= golang.org/x/net v0.39.0 h1:ZCu7HMWDxpXpaiKdhzIfaltL9Lp31x/3fCP11bc6/fY= golang.org/x/net v0.39.0/go.mod h1:X7NRbYVEA+ewNkCNyJ513WmMdQ3BineSwVtN2zD/d+E= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -192,8 +189,7 @@ golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= -golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20= golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -204,9 +200,8 @@ golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= -golang.org/x/term v0.30.0 h1:PQ39fJZ+mfadBm0y5WlL4vlM7Sx1Hgf13sMIY2+QS9Y= -golang.org/x/term v0.30.0/go.mod h1:NYYFdzHoI5wRh/h5tDMdMqCqPJZEuNqVR5xJLd/n67g= golang.org/x/term v0.31.0 h1:erwDkOK1Msy6offm1mOgvspSkslFnIGsFnxOKoufg3o= +golang.org/x/term v0.31.0/go.mod h1:R4BeIy7D95HzImkxGkTW1UQTtP54tio2RyHz7PwK0aw= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -217,8 +212,6 @@ golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= -golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= -golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0= golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= diff --git a/main.go b/main.go index 9d9468a..3b59bd5 100644 --- a/main.go +++ b/main.go @@ -27,10 +27,12 @@ func main() { // 'Assets' configures the asset server with the 'FS' variable pointing to the frontend files. // 'Bind' is a list of Go struct instances. The frontend has access to the methods of these instances. // 'Mac' options tailor the application when running an macOS. + appService := &App{} app := application.New(application.Options{ Name: "wails3-nuxt", Description: "A demo of using raw HTML & CSS", Services: []application.Service{ + application.NewService(appService), application.NewService(&GreetService{}), }, Assets: application.AssetOptions{ @@ -55,8 +57,10 @@ func main() { }, BackgroundColour: application.NewRGB(27, 38, 54), URL: "/", + Width: 1240, + Height: 850, }) - + appService.app = app // Create a goroutine that emits an event containing the current time every second. // The frontend can listen to this event and update the UI accordingly. go func() { diff --git a/services/site_service.go b/services/site_service.go index 1554288..81fb2f0 100644 --- a/services/site_service.go +++ b/services/site_service.go @@ -21,6 +21,13 @@ type RequestParams struct { URL string `json:"url"` } +type ResourcesList struct { + Script []string `json:"script"` + CSS []string `json:"css"` + Image []string `json:"image"` + Dom []string `json:"dom"` +} + // 获取当前页面的所有本站资源 js、css、img、链接 func (s SiteService) GetAllResources(rawURL string) ([]RequestParams, []string) { var RequestParamsAll []RequestParams @@ -43,14 +50,14 @@ func (s SiteService) GetAllResources(rawURL string) ([]RequestParams, []string) s.BaseDomain = parsed.Hostname() ActiveHref["https://"+host] = struct{}{} // 进入主站 - // browser := obj.Browser + browser := obj.Browser page := obj.Page s.LoopGet(page, PastHref, ActiveHref, &RequestParamsAll) for k := range PastHref { RouterAll = append(RouterAll, k) } list := deduplicationRequest(RequestParamsAll) - // 资源去重 + browser.Close() return list, RouterAll }