utilGenStaticFile.go

// Copyright (c) 2023 gpress Authors.
//
// This file is part of gpress.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

package main

import (
	"bufio"
	"bytes"
	"compress/gzip"
	"context"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"io"
	"io/fs"
	"net/http"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"sync"
	"time"

	"gitee.com/chunanyong/zorm"
	"golang.org/x/crypto/sha3"
)

// onlyOnce控制并发
// var onlyOnce = make(chan struct{}, 1)
var searchDataLock = &sync.Mutex{}
var genStaticHtmlLock = &sync.Mutex{}

// genSearchDataJson 生成flexSearch需要的json文件
func genSearchDataJson() error {
	//onlyOnce <- struct{}{}
	//defer func() { <-onlyOnce }()
	searchDataLock.Lock()
	defer searchDataLock.Unlock()

	finder := zorm.NewSelectFinder(tableContentName, "id,title,href_url,summary,create_time,tag,category_name,content,description").Append("WHERE status in (1,2) order by status desc, sortno desc")
	finder.SelectTotalCount = false
	//page := zorm.NewPage()
	//page.PageSize = 10000
	datas := make([]Content, 0)
	err := zorm.Query(context.Background(), finder, &datas, nil)
	if err != nil {
		return err
	}
	for i := 0; i < len(datas); i++ {
		if datas[i].HrefURL == "" {
			datas[i].HrefURL = funcBasePath() + funcTrimPrefixSlash(datas[i].Id)
		}
	}
	dataBytes, err := json.Marshal(datas)
	if err != nil {
		return err
	}
	err = os.WriteFile(searchDataJsonFile, dataBytes, os.ModePerm)
	if err != nil {
		return err
	}

	//压缩文件
	err = doGzipFile(searchDataJsonFile+compressedFileSuffix, bytes.NewReader(dataBytes))

	return err
}

// genStaticFile 生成全站静态文件和gzip文件,包括静态的html和search-data.json
func genStaticFile() error {
	reloadStatus = 1
	ctx := context.Background()
	//避免程序崩溃
	defer func() {
		//重置状态值
		reloadStatus = 0
		if r := recover(); r != nil {
			FuncLogPanic(ctx, fmt.Errorf("genStaticFile panic recovered: %v", r))
		}
	}()

	genStaticHtmlLock.Lock()
	defer genStaticHtmlLock.Unlock()

	//ctx := context.Background()
	contents := make([]Content, 0)

	f_post := zorm.NewSelectFinder(tableContentName, "id,tag").Append(" WHERE status<3 order by status desc, sortno desc")
	err := zorm.Query(ctx, f_post, &contents, nil)
	if err != nil {
		return err
	}
	//生成导航菜单的静态网页
	categoryIDs := make([]string, 0)
	f_category := zorm.NewSelectFinder(tableCategoryName, "id").Append(" WHERE status<3 order by status desc,sortno desc")
	err = zorm.Query(ctx, f_category, &categoryIDs, nil)
	if err != nil {
		return err
	}

	//删除整个目录
	//os.RemoveAll(staticHtmlDir)

	//删除deleteFileBeforeTime时间戳之前的老文件(当前时间后退 2 秒,避免误删刚生成的文件,linux生成文件 ModTime 的精度不够)
	deleteFileBeforeTime := time.Now().Add(-2 * time.Second)

	// 生成 default,pc,wap,weixin 等平台的静态文件
	useThemes := map[string]bool{}
	useThemes[""] = true
	err = genStaticFileByTheme(contents, categoryIDs, site.Theme, "")
	if err != nil {
		FuncLogError(ctx, err)
		//return err
	}
	useThemes[site.Theme] = true
	_, has := useThemes[site.ThemePC]
	//生成PC模板的静态网页
	if !has {
		err = genStaticFileByTheme(contents, categoryIDs, site.ThemePC, "Mozilla/5.0 (Windows NT 10.0; Win64; x64)")
		if err != nil {
			FuncLogError(ctx, err)
			//return err
		}
		useThemes[site.ThemePC] = true
	}

	// 生成手机WAP模板的静态网页
	_, has = useThemes[site.ThemeWAP]
	if !has {
		err = genStaticFileByTheme(contents, categoryIDs, site.ThemeWAP, "Mozilla/5.0 (Linux; Android 13;) Mobile")
		if err != nil {
			FuncLogError(ctx, err)
			//return err
		}
		useThemes[site.ThemeWAP] = true
	}
	//生成微信WX模板的静态网页
	_, has = useThemes[site.ThemeWX]
	if !has {
		err = genStaticFileByTheme(contents, categoryIDs, site.ThemeWX, "Mozilla/5.0 (Linux; Android 13;) Mobile MicroMessenger WeChat Weixin")
		if err != nil {
			FuncLogError(ctx, err)
			//return err
		}
		useThemes[site.ThemeWX] = true
	}

	// 重新生成 search-data.json
	err = genSearchDataJson()
	if err != nil {
		FuncLogError(ctx, err)
	}

	// 按照时间戳删除无效的文件
	err = filepath.WalkDir(staticHtmlDir, func(path string, d os.DirEntry, err error) error {
		if err != nil {
			return nil
		}
		// 删除空目录
		if d.IsDir() {
			entries, err := os.ReadDir(path) // 读取目录内容
			if err != nil {
				return err
			}
			if len(entries) == 0 {
				os.Remove(path)
			}
			return err
		}
		info, _ := d.Info()
		if info.ModTime().Before(deleteFileBeforeTime) {
			os.Remove(path) // 直接删除文件
		}
		return nil
	})

	return err
}

// genStaticFileByTheme 根据主题模板,生成静态文件
func genStaticFileByTheme(contents []Content, categories []string, theme string, userAgent string) error {
	// genMarkdownFile 是否生成markdown文件,如果主题模板中存在index.md,则生成markdown文件,否则不生成,默认值为false
	genMarkdownFile := pathExist(themeDir + site.Theme + "/index.md")
	domain := ""
	if site.Domain != "" {
		if strings.HasPrefix(site.Domain, "http://") || strings.HasPrefix(site.Domain, "https://") {
			domain = site.Domain
		} else { //默认使用https协议
			domain = "https://" + site.Domain
		}
	}
	tagsMap := make(map[string]bool, 0)
	//生成首页index网页
	fileHash, _, err := writeStaticHtml("", "", theme, userAgent, genMarkdownFile)
	if fileHash == "" || err != nil {
		return err
	}

	//创建sitemap.xml
	os.Remove(staticHtmlDir + theme + "/sitemap.xml")
	sitemapFile, err := os.OpenFile(staticHtmlDir+theme+"/sitemap.xml", os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0666)
	if err != nil {
		return err
	}
	defer sitemapFile.Close() // 确保在函数结束时关闭文件
	sitemapFile.WriteString(`<?xml version="1.0" encoding="UTF-8"?><urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">`)
	sitemapFile.WriteString("<url><loc>" + domain + "</loc></url>")
	if genMarkdownFile {
		sitemapFile.WriteString("<url><loc>" + domain + "/index.md</loc></url>")
	}

	//上一个分页
	prvePageFileHash := ""
	//生成文章的静态网页
	for i := 0; i < len(contents); i++ {
		tag := contents[i].Tag
		if tag != "" {
			tagsMap[tag] = true
		}
		//postURL := httpServerPath + "post/" + postId
		fileHash, success, err := writeStaticHtml(funcTrimPrefixSlash(contents[i].Id), "", theme, userAgent, genMarkdownFile)
		if fileHash == "" || err != nil {
			continue
		}
		if success {
			sitemapFile.WriteString("<url><loc>" + domain + funcBasePath() + funcTrimPrefixSlash(contents[i].Id) + "</loc></url>")
			if genMarkdownFile {
				sitemapFile.WriteString("<url><loc>" + domain + funcBasePath() + funcTrimPrefixSlash(contents[i].Id) + ".md</loc></url>")
			}
		}

		fileHash, success, err = writeStaticHtml("page/"+strconv.Itoa(i+1), prvePageFileHash, theme, userAgent, genMarkdownFile)
		if fileHash == "" || err != nil {
			continue
		}
		if success {
			sitemapFile.WriteString("<url><loc>" + domain + funcBasePath() + "page/" + strconv.Itoa(i+1) + "</loc></url>")
			if genMarkdownFile {
				sitemapFile.WriteString("<url><loc>" + domain + funcBasePath() + "page/" + strconv.Itoa(i+1) + ".md</loc></url>")
			}
		}
		//如果hash完全一致,认为是最后一页
		prvePageFileHash = fileHash
	}

	for i := 0; i < len(categories); i++ {
		//生成导航菜单首页index
		fileHash, success, err := writeStaticHtml(funcTrimSlash(categories[i]), "", theme, userAgent, genMarkdownFile)
		if fileHash == "" || err != nil {
			return err
		}
		if success {
			sitemapFile.WriteString("<url><loc>" + domain + funcBasePath() + funcTrimSlash(categories[i]) + "</loc></url>")
			if genMarkdownFile {
				sitemapFile.WriteString("<url><loc>" + domain + funcBasePath() + funcTrimSlash(categories[i]) + ".md</loc></url>")
			}
		}
		for j := 0; j < len(contents); j++ {
			fileHash, success, err := writeStaticHtml(funcTrimSlash(categories[i])+"/page/"+strconv.Itoa(j+1), prvePageFileHash, theme, userAgent, genMarkdownFile)
			if fileHash == "" || err != nil {
				continue
			}
			if success {
				sitemapFile.WriteString("<url><loc>" + domain + funcBasePath() + funcTrimSlash(categories[i]) + "/page/" + strconv.Itoa(j+1) + "</loc></url>")
				if genMarkdownFile {
					sitemapFile.WriteString("<url><loc>" + domain + funcBasePath() + funcTrimSlash(categories[i]) + "/page/" + strconv.Itoa(j+1) + ".md</loc></url>")
				}
			}
			//如果hash完全一致,认为是最后一页
			prvePageFileHash = fileHash
		}
	}

	//生成tag的静态页
	for tag := range tagsMap {
		//生成导航菜单首页index
		fileHash, success, err := writeStaticHtml("tag/"+tag, "", theme, userAgent, genMarkdownFile)
		if fileHash == "" || err != nil {
			return err
		}
		if success {
			sitemapFile.WriteString("<url><loc>" + domain + funcBasePath() + "tag/" + tag + "</loc></url>")
		}
		for j := 0; j < len(contents); j++ {
			fileHash, success, err := writeStaticHtml("tag/"+tag+"/page/"+strconv.Itoa(j+1), prvePageFileHash, theme, userAgent, genMarkdownFile)
			if fileHash == "" || err != nil {
				continue
			}
			if success {
				sitemapFile.WriteString("<url><loc>" + domain + funcBasePath() + "tag/" + tag + "/page/" + strconv.Itoa(j+1) + "</loc></url>")
				if genMarkdownFile {
					sitemapFile.WriteString("<url><loc>" + domain + funcBasePath() + "tag/" + tag + "/page/" + strconv.Itoa(j+1) + ".md</loc></url>")
				}
			}
			//如果hash完全一致,认为是最后一页
			prvePageFileHash = fileHash
		}
	}
	//结束写入sitemap文件
	sitemapFile.WriteString("</urlset>")

	//遍历当前使用的模板文件夹,压缩文本格式的文件
	err = filepath.WalkDir(templateDir+"theme/"+theme+"/", func(path string, info fs.DirEntry, err error) error {
		if err != nil {
			return err
		}
		// 分隔符统一为 / 斜杠
		path = filepath.ToSlash(path)

		// 只处理 js 和 css 文件夹
		if !(strings.Contains(path, "/js/") || strings.Contains(path, "/css/")) {
			return nil
		}

		//获取文件后缀
		suffix := filepath.Ext(path)

		// 压缩 js,mjs,json,css,html
		// 压缩字体文件 ttf,otf,svg  gzip_types font/ttf font/otf image/svg+xml
		if !(suffix == ".js" || suffix == ".mjs" || suffix == ".json" || suffix == ".css" || suffix == ".html" || suffix == ".ttf" || suffix == ".otf" || suffix == ".svg") {
			return nil
		}

		// 获取要打包的文件信息
		readFile, err := os.Open(path)
		if err != nil {
			return err
		}
		defer readFile.Close()
		reader := bufio.NewReader(readFile)
		//压缩文件
		err = doGzipFile(path+compressedFileSuffix, reader)

		return err
	})
	return err
}

// writeStaticHtml 写入静态html
func writeStaticHtml(urlFilePath string, fileHash string, theme string, userAgent string, genMarkdownFile bool) (string, bool, error) {
	httpurl := httpServerPath + urlFilePath
	markdownHttpurl := httpurl + "/index.md"

	filePath := staticHtmlDir + theme + funcBasePath() + urlFilePath
	markdownFilePath := staticHtmlDir + theme + "/_markdown" + funcBasePath() + "index.md"
	if urlFilePath != "" {
		markdownHttpurl = httpurl + ".md"
		markdownFilePath = staticHtmlDir + theme + "/_markdown" + funcBasePath() + urlFilePath + ".md"
		filePath = filePath + "/"
	}

	body, err := responseBodyBytes(httpurl, userAgent)
	if err != nil {
		return "", false, err
	}
	//计算hash
	bytehex := sha3.Sum256(body)
	bodyHash := hex.EncodeToString(bytehex[:])
	if bodyHash == fileHash { //如果hash一致,不再生成文件
		return bodyHash, false, nil
	}
	// 写入文件
	os.MkdirAll(filePath, os.ModePerm)
	err = os.WriteFile(filePath+"index.html", body, os.ModePerm)
	if err != nil {
		return bodyHash, false, err
	}
	// gzip 压缩 html 文件
	err = doGzipFile(filePath+"index.html"+compressedFileSuffix, bytes.NewReader(body))
	if err != nil {
		return bodyHash, false, err
	}
	if !genMarkdownFile {
		return bodyHash, true, err
	}

	// 生成 markdown文件
	markdown, err := responseBodyBytes(markdownHttpurl, userAgent)
	if err != nil {
		return "", false, err
	}
	// 写入文件
	os.MkdirAll(filepath.Dir(markdownFilePath), os.ModePerm)
	err = os.WriteFile(markdownFilePath, markdown, os.ModePerm)
	if err != nil {
		return bodyHash, false, err
	}
	// gzip 压缩 markdown 文件
	err = doGzipFile(markdownFilePath+compressedFileSuffix, bytes.NewReader(markdown))
	if err != nil {
		return bodyHash, false, err
	}

	return bodyHash, true, nil
}

// doGzipFile 压缩gzip文件
func doGzipFile(gzipFilePath string, reader io.Reader) error {

	//如果文件存在就删除
	if pathExist(gzipFilePath) {
		os.Remove(gzipFilePath)
	}
	//创建文件
	gzipFile, err := os.Create(gzipFilePath)
	if err != nil {
		return err
	}
	defer gzipFile.Close()

	gzipWrite, err := gzip.NewWriterLevel(gzipFile, gzip.BestCompression)
	if err != nil {
		return err
	}
	defer gzipWrite.Close()
	_, err = io.Copy(gzipWrite, reader)
	return err
}

// responseBodyBytes 获取http资源的body字节数据
func responseBodyBytes(httpurl string, userAgent string) ([]byte, error) {
	client := &http.Client{}
	req, err := http.NewRequest("GET", httpurl, nil)
	if err != nil {

		return nil, err
	}

	// 设置请求头
	if userAgent != "" {
		req.Header.Set("User-Agent", userAgent)
	}
	response, err := client.Do(req)
	if err != nil {
		return nil, err
	}
	defer response.Body.Close()

	// 读取资源数据 body: []byte
	body, err := io.ReadAll(response.Body)
	// 关闭资源流
	response.Body.Close()
	return body, err
}