us-coach-spider/product/controller.go

package product

import (
	"context"
	"fmt"
	"gitea.timerzz.com/kedaya_haitao/coach-spider/database"
	coach_client "gitea.timerzz.com/kedaya_haitao/coach-spider/pkg/coach-client"
	"github.com/samber/lo"
	"gorm.io/gorm"
	"gorm.io/gorm/clause"
	"log/slog"
	"time"
)

type Controller struct {
	ctx        context.Context
	client     *coach_client.Client
	updateTime time.Time // 上次抓取时间
	db         *gorm.DB
	Option
}

func NewController(client *coach_client.Client, db *gorm.DB) *Controller {
	ctl := &Controller{
		client: client,
		db:     db,
	}
	ctl.AutoMigrate()
	ctl.LoadOption()
	return ctl
}

func (c *Controller) AutoMigrate() {
	if err := c.db.AutoMigrate(&database.Product{}, &database.HistoryPrice{}, &Option{}); err != nil {
		panic(err)
	}
}
func (c *Controller) Run(ctx context.Context) {
	c.ctx = ctx
	ticker := time.NewTicker(c.Interval)
	if err := c.Crawl(); err != nil {
		slog.Error(err.Error())
	} else {
		slog.Info("抓取信息成功")
		c.updateTime = time.Now()
	}
	for {
		select {
		case <-ctx.Done():
			return
		case <-ticker.C:
			if err := c.Crawl(); err != nil {
				slog.Error(err.Error())
			} else {
				slog.Info("抓取信息成功")
				c.updateTime = time.Now()
			}
		}

	}
}

func (c *Controller) Crawl() error {
	slog.Info("开始抓取信息")
	for page, totalPage := 1, -1; page <= totalPage || totalPage == -1; page++ {
		resp, err := c.client.ViewAllBags(c.ctx, page)
		if err != nil {
			return fmt.Errorf("访问coach第%d页失败: %w", page, err)
		}
		totalPage = resp.PageData.TotalPages

		if err = c.saveRespData(resp.PageData.Products); err != nil {
			return fmt.Errorf("保存第%d页数据失败: %w", page, err)
		}
	}
	return nil
}

func (c *Controller) saveRespData(list []coach_client.Product) error {
	var products = make([]database.Product, 0, len(list))
	for _, resp := range list {
		for _, color := range resp.Colors {
			price, _ := lo.Find(resp.VariantsOnSale, func(item coach_client.Variant) bool {
				return item.Id == color.VgId
			})
			products = append(products, database.Product{
				Name:         resp.Name,
				Pid:          color.VgId,
				Color:        color.Text,
				Link:         fmt.Sprintf("%s/%s", "https://www.coachoutlet.com", color.Url),
				Image:        color.Media.Thumbnail.Src,
				Orderable:    price.OnSale,
				USPrice:      price.Price.Sales.Value,
				Freight:      c.Freight,
				ExchangeRate: c.ExchangeRate,
			})
		}
	}
	// 去重
	products = lo.UniqBy(products, func(p database.Product) string {
		return p.Pid
	})
	return c.db.Clauses(clause.OnConflict{
		Columns:   []clause.Column{{Name: "pid"}},
		DoUpdates: clause.AssignmentColumns([]string{"name", "color", "link", "orderable", "us_price", "cny_price", "cal_mark", "freight", "rate"}),
	}).Create(products).Error
}