cn-coach-spider/spider/controller.go

package spider

import (
	"context"
	"fmt"
	"gitea.timerzz.com/kedaya_haitao/common/model/product"
	coach_client "gitea.timerzz.com/kedaya_haitao/common/pkg/coach-client"
	"github.com/golang/glog"
	"github.com/samber/lo"
	"gorm.io/gorm"
	"gorm.io/gorm/clause"
	"time"
)

type Controller struct {
	ctx      context.Context
	client   *coach_client.CN
	db       *gorm.DB
	interval time.Duration
}

func NewController(client *coach_client.CN, db *gorm.DB, interval time.Duration) *Controller {
	ctl := &Controller{
		client:   client,
		db:       db,
		interval: interval,
	}
	ctl.AutoMigrate()
	return ctl
}

func (c *Controller) AutoMigrate() {
	if err := c.db.AutoMigrate(&productv1.Product{}, &productv1.HistoryPrice{}); err != nil {
		panic(err)
	}
}
func (c *Controller) Run(ctx context.Context) {
	c.ctx = ctx
	ticker := time.NewTicker(c.interval)
	defer ticker.Stop()

	if err := c.Crawl(); err != nil {
		glog.Error(err.Error())
	} else {
		glog.Info("抓取信息成功")
	}
	for {
		select {
		case <-ctx.Done():
			return
		case <-ticker.C:
			if err := c.Crawl(); err != nil {
				glog.Error(err.Error())
			} else {
				glog.Info("抓取信息成功")
			}
		}

	}
}

func (c *Controller) Crawl() error {
	glog.Info("开始抓取信息")
	for page, totalPage := 1, 1; page <= totalPage; page++ {
		resp, err := c.client.ListItems(c.ctx, page, 50)
		if err != nil {
			return fmt.Errorf("访问coach第%d页失败: %w", page, err)
		}
		totalPage = resp.TotalPages

		if err = c.saveRespData(resp.Items); err != nil {
			return fmt.Errorf("保存第%d页数据失败: %w", page, err)
		}
		glog.Infof("第%d页数据保存成功", page)
	}
	return nil
}

func (c *Controller) saveRespData(list []coach_client.CNItem) error {
	var products = make([]productv1.Product, 0, len(list))
	for _, item := range list {
		products = append(products, productv1.Product{
			UpdatedAt:   time.Now(),
			Name:        item.Title,
			Pid:         item.Code,
			Link:        fmt.Sprintf("https://www.coachoutlet.cn/products/%s", item.Code),
			Image:       item.Images[0].Imgs[0].Img,
			Orderable:   item.Stock > 0,
			DiscPercent: 100 - int(item.DiscountRateMin*100),
			CNYPrice:    item.SkuMaxPrice,
			Website:     productv1.WebSite_CN_Coach,
		})
	}
	// 去重
	products = lo.UniqBy(products, func(p productv1.Product) string {
		return p.Pid
	})
	return c.db.Clauses(clause.OnConflict{
		Columns:   []clause.Column{{Name: "pid"}},
		DoUpdates: clause.AssignmentColumns([]string{"name", "link", "orderable", "cny_price", "rate", "price_status", "disc_percent", "updated_at"}),
	}).Create(products).Error
}