修改下载页面渲染

2025-11-24 18:59:19 +08:00
parent 51cfd5cf21
commit 65d4d458d0
10 changed files with 2328 additions and 111 deletions
--- a/utils/file.go
+++ b/utils/file.go
@@ -1,8 +1,10 @@
 package utils

 import (
+	"bytes"
 	"fmt"
 	"go-site-clone/config"
+	"go-site-clone/types"
 	"io"
 	"log"
 	"net/http"
@@ -10,6 +12,8 @@ import (
 	"os"
 	"path"
 	"path/filepath"
+	"regexp"
+	"strings"
 	"time"
 )

@@ -24,43 +28,127 @@ type FileDir struct {

 // 下载文件到本地
 func (*File) Download(uri string) string {
-	// 解析文件链接及 路径
+	return downloadWithRetry(uri, false, 3, nil)
+}
+
+// DownloadWithOptions 带配置选项的下载
+func (*File) DownloadWithOptions(uri string, options *types.DownloadOptions) string {
+	return downloadWithRetry(uri, false, 3, options)
+}
+
+// downloadWithRetry 带重试的下载函数
+func downloadWithRetry(uri string, isHTML bool, maxRetries int, options *types.DownloadOptions) string {
+	// 如果没有提供配置，使用默认配置
+	if options == nil {
+		defaultOpts := types.DefaultDownloadOptions()
+		options = &defaultOpts
+	}
+	// 解析文件链接及路径
 	u, err := url.Parse(uri)
 	if err != nil {
+		log.Printf("URL解析失败 %s: %v", uri, err)
 		return ""
 	}
+
 	host := u.Hostname()
 	if u.Port() != "" {
-		host += u.Port()
+		host += ":" + u.Port()
 	}
+
 	// 获取路径部分
 	filePath := u.Path
+	if isHTML && (filePath == "" || filePath == "/") {
+		filePath = "/index.html"
+	} else if isHTML && path.Base(filePath) == path.Dir(filePath) {
+		filePath = filePath + "/index.html"
+	}
+
 	appConfig, _ := config.LoadConfig()
 	fp := filepath.Join(appConfig.SiteFileDir, host, filePath)
-	// 获取文件名
-	// fileName := path.Base(filePath)
-	log.Println("开始下载：", uri)
-	resp, err := http.Get(uri)
-	if err != nil {
-		fmt.Println("下载失败:", err)
-		return ""
+
+	// 检查文件是否已存在
+	if _, err := os.Stat(fp); err == nil {
+		log.Printf("文件已存在，跳过: %s", fp)
+		return fp
 	}
-	defer resp.Body.Close()
-	// 创建本地文件
-	outFile, err := CreateFileWithDirs(fp)
-	if err != nil {
-		fmt.Println("文件创建失败:", err)
-		return ""
+
+	// 重试逻辑
+	var lastErr error
+	for attempt := 0; attempt < maxRetries; attempt++ {
+		if attempt > 0 {
+			log.Printf("重试下载 (%d/%d): %s", attempt+1, maxRetries, uri)
+			time.Sleep(time.Second * time.Duration(attempt)) // 递增延迟
+		}
+
+		// 创建带超时的HTTP客户端
+		client := &http.Client{
+			Timeout: 30 * time.Second,
+		}
+
+		resp, err := client.Get(uri)
+		if err != nil {
+			lastErr = fmt.Errorf("HTTP请求失败: %w", err)
+			continue
+		}
+
+		if resp.StatusCode != http.StatusOK {
+			resp.Body.Close()
+			lastErr = fmt.Errorf("HTTP状态码错误: %d", resp.StatusCode)
+			continue
+		}
+
+		// 创建本地文件
+		outFile, err := CreateFileWithDirs(fp)
+		if err != nil {
+			resp.Body.Close()
+			lastErr = fmt.Errorf("文件创建失败: %w", err)
+			continue
+		}
+
+		// 如果是HTML文件，需要替换资源路径
+		if isHTML {
+			// 读取HTML内容
+			htmlContent, err := io.ReadAll(resp.Body)
+			resp.Body.Close()
+
+			if err != nil {
+				outFile.Close()
+				os.Remove(fp)
+				lastErr = fmt.Errorf("读取HTML内容失败: %w", err)
+				continue
+			}
+
+			// 替换资源路径
+			modifiedHTML := replaceHTMLResourcePaths(string(htmlContent), uri, options)
+
+			// 写入修改后的HTML
+			_, err = outFile.WriteString(modifiedHTML)
+			outFile.Close()
+
+			if err != nil {
+				os.Remove(fp)
+				lastErr = fmt.Errorf("写入HTML文件失败: %w", err)
+				continue
+			}
+		} else {
+			// 非 HTML 文件，直接写入
+			_, err = io.Copy(outFile, resp.Body)
+			resp.Body.Close()
+			outFile.Close()
+
+			if err != nil {
+				os.Remove(fp) // 删除不完整的文件
+				lastErr = fmt.Errorf("写入文件失败: %w", err)
+				continue
+			}
+		}
+
+		log.Printf("下载完成: %s", fp)
+		return fp
 	}
-	defer outFile.Close()
-	// 将响应内容写入文件
-	_, err = io.Copy(outFile, resp.Body)
-	if err != nil {
-		fmt.Println("写入失败:", err)
-		return ""
-	}
-	fmt.Println("下载完成:", fp)
-	return fp
+
+	log.Printf("下载失败（已重试%d次）%s: %v", maxRetries, uri, lastErr)
+	return ""
 }

 // 创建目录
@@ -84,47 +172,233 @@ func CreateFileWithDirs(filePath string) (*os.File, error) {

 // 下载html文件到本地
 func (*File) HTMLDownload(uri string) string {
-	// 解析文件链接及 路径
-	u, err := url.Parse(uri)
-	if err != nil {
-		return ""
-	}
-	host := u.Hostname()
-	if u.Port() != "" {
-		host += u.Port()
-	}
-	// 获取路径部分
-	filePath := u.Path
-	appConfig, _ := config.LoadConfig()
-	// 获取文件名
-	fileName := path.Base(filePath)
-	if fileName == "/" || filePath == "" {
-		filePath = filePath + "/index.html"
-	}
-	fp := filepath.Join(appConfig.SiteFileDir, host, filePath)
+	return downloadWithRetry(uri, true, 3, nil)
+}

-	log.Println("开始下载：", uri)
-	resp, err := http.Get(uri)
+// HTMLDownloadWithOptions 带配置选项的HTML下载
+func (*File) HTMLDownloadWithOptions(uri string, options *types.DownloadOptions) string {
+	return downloadWithRetry(uri, true, 3, options)
+}
+
+// replaceHTMLResourcePaths 替换HTML中的资源路径为相对路径
+func replaceHTMLResourcePaths(htmlContent string, baseURL string, options *types.DownloadOptions) string {
+	if options == nil {
+		defaultOpts := types.DefaultDownloadOptions()
+		options = &defaultOpts
+	}
+	parsedBase, err := url.Parse(baseURL)
 	if err != nil {
-		fmt.Println("下载失败:", err)
+		return htmlContent
+	}
+
+	baseHost := parsedBase.Hostname()
+	if parsedBase.Port() != "" {
+		baseHost += ":" + parsedBase.Port()
+	}
+	baseScheme := parsedBase.Scheme
+	basePath := parsedBase.Path
+
+	// 如果基础路径不是以/结尾，去掉文件名部分
+	if basePath != "" && basePath != "/" {
+		basePath = path.Dir(basePath)
+	}
+	if basePath == "." {
+		basePath = "/"
+	}
+
+	modified := htmlContent
+
+	// 1. 替换 CSS 链接: <link href="..." rel="stylesheet">
+	cssRegex := regexp.MustCompile(`(<link[^>]*?href=["'])([^"']+)(["'][^>]*?>)`)
+	modified = cssRegex.ReplaceAllStringFunc(modified, func(match string) string {
+		parts := cssRegex.FindStringSubmatch(match)
+		if len(parts) == 4 {
+			originalURL := parts[2]
+			if newPath := convertToRelativePath(originalURL, baseURL, baseHost, baseScheme, basePath, "css", options); newPath != "" {
+				return parts[1] + newPath + parts[3]
+			}
+		}
+		return match
+	})
+
+	// 2. 替换 JavaScript: <script src="...">
+	scriptRegex := regexp.MustCompile(`(<script[^>]*?src=["'])([^"']+)(["'][^>]*?>)`)
+	modified = scriptRegex.ReplaceAllStringFunc(modified, func(match string) string {
+		parts := scriptRegex.FindStringSubmatch(match)
+		if len(parts) == 4 {
+			originalURL := parts[2]
+			if newPath := convertToRelativePath(originalURL, baseURL, baseHost, baseScheme, basePath, "script", options); newPath != "" {
+				return parts[1] + newPath + parts[3]
+			}
+		}
+		return match
+	})
+
+	// 3. 替换图片: <img src="...">
+	imgRegex := regexp.MustCompile(`(<img[^>]*?src=["'])([^"']+)(["'][^>]*?>)`)
+	modified = imgRegex.ReplaceAllStringFunc(modified, func(match string) string {
+		parts := imgRegex.FindStringSubmatch(match)
+		if len(parts) == 4 {
+			originalURL := parts[2]
+			if newPath := convertToRelativePath(originalURL, baseURL, baseHost, baseScheme, basePath, "image", options); newPath != "" {
+				return parts[1] + newPath + parts[3]
+			}
+		}
+		return match
+	})
+
+	// 4. 替换视频: <video src="..."> 和 <source src="...">
+	videoRegex := regexp.MustCompile(`(<(?:video|source)[^>]*?src=["'])([^"']+)(["'][^>]*?>)`)
+	modified = videoRegex.ReplaceAllStringFunc(modified, func(match string) string {
+		parts := videoRegex.FindStringSubmatch(match)
+		if len(parts) == 4 {
+			originalURL := parts[2]
+			if newPath := convertToRelativePath(originalURL, baseURL, baseHost, baseScheme, basePath, "video", options); newPath != "" {
+				return parts[1] + newPath + parts[3]
+			}
+		}
+		return match
+	})
+
+	// 5. 替换音频: <audio src="...">
+	audioRegex := regexp.MustCompile(`(<audio[^>]*?src=["'])([^"']+)(["'][^>]*?>)`)
+	modified = audioRegex.ReplaceAllStringFunc(modified, func(match string) string {
+		parts := audioRegex.FindStringSubmatch(match)
+		if len(parts) == 4 {
+			originalURL := parts[2]
+			if newPath := convertToRelativePath(originalURL, baseURL, baseHost, baseScheme, basePath, "video", options); newPath != "" {
+				return parts[1] + newPath + parts[3]
+			}
+		}
+		return match
+	})
+
+	// 6. 替换 CSS 中的 url(): url("...") 或 url('...') 或 url(...)
+	cssURLRegex := regexp.MustCompile(`(url\(["']?)([^"')]+)(["']?\))`)
+	modified = cssURLRegex.ReplaceAllStringFunc(modified, func(match string) string {
+		parts := cssURLRegex.FindStringSubmatch(match)
+		if len(parts) == 4 {
+			originalURL := parts[2]
+			// 跳过 data: 和 # 开头的URL
+			if !strings.HasPrefix(originalURL, "data:") && !strings.HasPrefix(originalURL, "#") {
+				if newPath := convertToRelativePath(originalURL, baseURL, baseHost, baseScheme, basePath, "image", options); newPath != "" {
+					return parts[1] + newPath + parts[3]
+				}
+			}
+		}
+		return match
+	})
+
+	return modified
+}
+
+// convertToRelativePath 将绝对URL转换为相对路径
+func convertToRelativePath(resourceURL, baseURL, baseHost, baseScheme, basePath, resourceType string, options *types.DownloadOptions) string {
+	// 跳过空值、data:, javascript:, mailto:, # 等
+	if resourceURL == "" ||
+		strings.HasPrefix(resourceURL, "data:") ||
+		strings.HasPrefix(resourceURL, "javascript:") ||
+		strings.HasPrefix(resourceURL, "mailto:") ||
+		strings.HasPrefix(resourceURL, "#") {
 		return ""
 	}
-	defer resp.Body.Close()
-	// 创建本地文件
-	outFile, err := CreateFileWithDirs(fp)
-	if err != nil {
-		fmt.Println("文件创建失败:", err)
+
+	// 如果已经是相对路径，不处理
+	if !strings.HasPrefix(resourceURL, "http://") &&
+		!strings.HasPrefix(resourceURL, "https://") &&
+		!strings.HasPrefix(resourceURL, "//") {
 		return ""
 	}
-	defer outFile.Close()
-	// 将响应内容写入文件
-	_, err = io.Copy(outFile, resp.Body)
+
+	// 处理 // 开头的URL
+	if strings.HasPrefix(resourceURL, "//") {
+		resourceURL = baseScheme + ":" + resourceURL
+	}
+
+	// 解析资源URL
+	parsedResource, err := url.Parse(resourceURL)
 	if err != nil {
-		fmt.Println("写入失败:", err)
 		return ""
 	}
-	fmt.Println("下载完成:", fp)
-	return fp
+
+	resourceHost := parsedResource.Hostname()
+	if parsedResource.Port() != "" {
+		resourceHost += ":" + parsedResource.Port()
+	}
+
+	// 使用配置判断是否应该下载该资源
+	if !options.ShouldDownloadExternal(resourceHost, baseHost, resourceType) {
+		// 不下载外部资源，保持原URL
+		return ""
+	} // 获取资源路径
+	resourcePath := parsedResource.Path
+	if resourcePath == "" {
+		resourcePath = "/"
+	}
+
+	// 计算相对路径
+	relativePath := calculateRelativePath(basePath, resourcePath)
+
+	log.Printf("路径替换: %s -> %s", resourceURL, relativePath)
+
+	return relativePath
+}
+
+// calculateRelativePath 计算从 basePath 到 targetPath 的相对路径
+func calculateRelativePath(basePath, targetPath string) string {
+	if basePath == "" {
+		basePath = "/"
+	}
+	if targetPath == "" {
+		targetPath = "/"
+	}
+
+	// 分割路径
+	baseParts := strings.Split(strings.Trim(basePath, "/"), "/")
+	targetParts := strings.Split(strings.Trim(targetPath, "/"), "/")
+
+	// 找到公共前缀
+	commonLen := 0
+	for i := 0; i < len(baseParts) && i < len(targetParts); i++ {
+		if baseParts[i] == targetParts[i] {
+			commonLen++
+		} else {
+			break
+		}
+	}
+
+	// 构建相对路径
+	var result bytes.Buffer
+
+	// 添加 ../ 返回上级目录
+	upLevels := len(baseParts) - commonLen
+	for i := 0; i < upLevels; i++ {
+		if i > 0 {
+			result.WriteString("/")
+		}
+		result.WriteString("..")
+	}
+
+	// 添加目标路径的剩余部分
+	for i := commonLen; i < len(targetParts); i++ {
+		if result.Len() > 0 {
+			result.WriteString("/")
+		}
+		result.WriteString(targetParts[i])
+	}
+
+	// 如果结果为空，返回当前目录
+	if result.Len() == 0 {
+		return "./"
+	}
+
+	// 如果不是以 . 或 / 开头，添加 ./
+	resultStr := result.String()
+	if !strings.HasPrefix(resultStr, ".") && !strings.HasPrefix(resultStr, "/") {
+		return "./" + resultStr
+	}
+
+	return resultStr
 }

 // 获取文件夹列表