package spider import ( "context" "fmt" "gitea.timerzz.com/kedaya_haitao/cn-coach-spider/pkg/options" "gitea.timerzz.com/kedaya_haitao/common/model/product" coach_client "gitea.timerzz.com/kedaya_haitao/common/pkg/coach-client" "github.com/golang/glog" "github.com/samber/lo" "gorm.io/gorm" "gorm.io/gorm/clause" "time" ) type Controller struct { ctx context.Context client *coach_client.CN db *gorm.DB interval time.Duration website productv1.Website linkPrefix string pid string calculates []productv1.CalculateProcess } func NewController(client *coach_client.CN, db *gorm.DB, interval time.Duration, shopType string) *Controller { ctl := &Controller{ client: client, db: db, interval: interval, website: productv1.WebSite_CN_Coach, linkPrefix: "https://www.coach.com.cn/products/", pid: CNCoachPid, } if shopType == options.ShopType_outlet { ctl.website = productv1.WebSite_CN_Coach_Outlet ctl.linkPrefix = "https://www.coach.com.cn/outlet/products/" ctl.pid = CNCoachOutletPid } ctl.AutoMigrate() ctl.LoadCalculateProcess() return ctl } func (c *Controller) AutoMigrate() { if err := c.db.AutoMigrate(&productv1.Product{}, &productv1.HistoryPrice{}); err != nil { panic(err) } } func (c *Controller) Run(ctx context.Context) { c.ctx = ctx ticker := time.NewTicker(c.interval) defer ticker.Stop() if err := c.Crawl(); err != nil { glog.Error(err.Error()) } else { glog.Info("抓取信息成功") } for { select { case <-ctx.Done(): return case <-ticker.C: if err := c.Crawl(); err != nil { glog.Error(err.Error()) } else { glog.Info("抓取信息成功") } } } } func (c *Controller) Crawl() error { glog.Info("开始抓取信息") for page, totalPage := 1, 1; page <= totalPage; page++ { resp, err := c.client.ListItems(c.ctx, page, 50) if err != nil { return fmt.Errorf("访问coach第%d页失败: %w", page, err) } totalPage = resp.TotalPages if err = c.saveRespData(resp.Items); err != nil { return fmt.Errorf("保存第%d页数据失败: %w", page, err) } glog.Infof("第%d页数据保存成功", page) } return nil } func (c *Controller) saveRespData(list []coach_client.CNItem) error { var products = make([]productv1.Product, 0, len(list)) for _, item := range list { var savedProduct productv1.Product c.db.Model(&savedProduct).Where("pid = ?", item.Code).Select("dw_price").Scan(&savedProduct) p := productv1.Product{ UpdatedAt: time.Now(), Name: item.Title, Pid: item.Code, Link: fmt.Sprintf("%s%s", c.linkPrefix, item.Code), Image: item.Images[0].Imgs[0].Img, Orderable: item.Stock > 0, DiscPercent: 100 - int(item.DiscountRateMin*100), OriginalPrice: item.SkuMaxPrice, Website: c.website, DWPrice: savedProduct.DWPrice, } var calculate []productv1.CalculateProcess c.db.Model(&productv1.CalculateProcess{}).Find(&calculate, "pid = ? AND website = ?", p.Pid, c.website) p.CalCNY(append(calculate, c.calculates...)) products = append(products, p) } // 去重 products = lo.UniqBy(products, func(p productv1.Product) string { return p.Pid }) return c.db.Clauses(clause.OnConflict{ Columns: []clause.Column{{Name: "pid"}}, DoUpdates: clause.AssignmentColumns([]string{"name", "link", "orderable", "original_price", "rate", "price_status", "disc_percent", "updated_at"}), }).Create(products).Error }