cn-coach-spider/spider/controller.go

package spider

import (
	"context"
	"fmt"
	"gitea.timerzz.com/kedaya_haitao/cn-coach-spider/pkg/options"
	"gitea.timerzz.com/kedaya_haitao/common/model/product"
	coach_client "gitea.timerzz.com/kedaya_haitao/common/pkg/coach-client"
	"github.com/golang/glog"
	"github.com/samber/lo"
	"gorm.io/gorm"
	"gorm.io/gorm/clause"
	"time"
)

type Controller struct {
	ctx        context.Context
	client     *coach_client.CN
	db         *gorm.DB
	interval   time.Duration
	website    productv1.Website
	linkPrefix string
	pid        string

	calculates []productv1.CalculateProcess
}

func NewController(client *coach_client.CN, db *gorm.DB, interval time.Duration, shopType string) *Controller {
	ctl := &Controller{
		client:     client,
		db:         db,
		interval:   interval,
		website:    productv1.WebSite_CN_Coach,
		linkPrefix: "https://www.coach.com.cn/products/",
		pid:        CNCoachPid,
	}
	if shopType == options.ShopType_outlet {
		ctl.website = productv1.WebSite_CN_Coach_Outlet
		ctl.linkPrefix = "https://www.coach.com.cn/outlet/products/"
		ctl.pid = CNCoachOutletPid
	}
	ctl.AutoMigrate()
	ctl.LoadCalculateProcess()
	return ctl
}

func (c *Controller) AutoMigrate() {
	if err := c.db.AutoMigrate(&productv1.Product{}, &productv1.HistoryPrice{}); err != nil {
		panic(err)
	}
}

func (c *Controller) Run(ctx context.Context) {
	c.ctx = ctx
	ticker := time.NewTicker(c.interval)
	defer ticker.Stop()

	if err := c.Crawl(); err != nil {
		glog.Error(err.Error())
	} else {
		glog.Info("抓取信息成功")
	}
	for {
		select {
		case <-ctx.Done():
			return
		case <-ticker.C:
			if err := c.Crawl(); err != nil {
				glog.Error(err.Error())
			} else {
				glog.Info("抓取信息成功")
			}
		}

	}
}

func (c *Controller) Crawl() error {
	glog.Info("开始抓取信息")
	for page, totalPage := 1, 1; page <= totalPage; page++ {
		resp, err := c.client.ListItems(c.ctx, page, 50)
		if err != nil {
			return fmt.Errorf("访问coach第%d页失败: %w", page, err)
		}
		totalPage = resp.TotalPages

		if err = c.saveRespData(resp.Items); err != nil {
			return fmt.Errorf("保存第%d页数据失败: %w", page, err)
		}
		glog.Infof("第%d页数据保存成功", page)
	}
	return nil
}

func (c *Controller) saveRespData(list []coach_client.CNItem) error {
	var products = make([]productv1.Product, 0, len(list))
	for _, item := range list {
		var savedProduct productv1.Product
		c.db.Model(&savedProduct).Where("pid = ?", item.Code).Select("dw_price").Scan(&savedProduct)
		p := productv1.Product{
			UpdatedAt:     time.Now(),
			Name:          item.Title,
			Pid:           item.Code,
			Link:          fmt.Sprintf("%s%s", c.linkPrefix, item.Code),
			Image:         item.Images[0].Imgs[0].Img,
			Orderable:     item.Stock > 0,
			DiscPercent:   100 - int(item.DiscountRateMin*100),
			OriginalPrice: item.SkuMaxPrice,
			Website:       c.website,
			DWPrice:       savedProduct.DWPrice,
		}
		var calculate []productv1.CalculateProcess
		c.db.Model(&productv1.CalculateProcess{}).Find(&calculate, "pid = ? AND website = ?", p.Pid, c.website)
		p.CalCNY(append(calculate, c.calculates...))
		products = append(products, p)
	}
	// 去重
	products = lo.UniqBy(products, func(p productv1.Product) string {
		return p.Pid
	})
	return c.db.Clauses(clause.OnConflict{
		Columns:   []clause.Column{{Name: "pid"}},
		DoUpdates: clause.AssignmentColumns([]string{"name", "link", "orderable", "original_price", "rate", "price_status", "disc_percent", "updated_at"}),
	}).Create(products).Error
}