From d55cdfc0d57d5d8a37b0708f81cba331e8d69110 Mon Sep 17 00:00:00 2001 From: timerzz Date: Sun, 16 Jun 2024 11:08:28 +0800 Subject: [PATCH] =?UTF-8?q?feat=20=E6=94=AF=E6=8C=81coach=E5=92=8Coutlet?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cmd/main.go | 15 +++++- go.mod | 3 +- go.sum | 6 ++- pkg/options/shop.go | 6 +++ server/spider.go | 83 +++++++++++++++++++++++++++++++ spider/calculate.go | 115 +++++++++++++++++++++++++++++++++++++++++++ spider/controller.go | 63 ++++++++++++++++-------- 7 files changed, 266 insertions(+), 25 deletions(-) create mode 100644 pkg/options/shop.go create mode 100644 server/spider.go create mode 100644 spider/calculate.go diff --git a/cmd/main.go b/cmd/main.go index 424b96d..9d4d653 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -3,6 +3,7 @@ package main import ( "context" "gitea.timerzz.com/kedaya_haitao/cn-coach-spider/pkg/options" + "gitea.timerzz.com/kedaya_haitao/cn-coach-spider/server" "gitea.timerzz.com/kedaya_haitao/cn-coach-spider/spider" coach_client "gitea.timerzz.com/kedaya_haitao/common/pkg/coach-client" "gitea.timerzz.com/kedaya_haitao/common/pkg/database" @@ -38,13 +39,21 @@ func main() { Colorful: true, }, ) + shopType := os.Getenv("SHOP_TYPE") + if shopType != options.ShopType_coach && shopType != options.ShopType_outlet { + glog.Fatalf("SHOP_TYPE环境变量错误，请设置SHOP_TYPE=coach或SHOP_TYPE=outlet") + } // coach client - client, err := coach_client.CNClient() + var xMac = coach_client.XMac_Coach + if shopType == options.ShopType_outlet { + xMac = coach_client.XMac_Outlet + } + client, err := coach_client.CNClient(xMac) if err != nil { glog.Fatalf("初始化coach client失败：%v", err) } - spiderCtl := spider.NewController(client, db, cfg.Interval) + spiderCtl := spider.NewController(client, db, cfg.Interval, shopType) go spiderCtl.Run(ctx) @@ -54,7 +63,9 @@ func main() { r.Get("/health", func(ctx fiber.Ctx) error { return ctx.SendString("ok") }) + api := r.Group("/api/v1") + server.NewSpiderSvc(spiderCtl, shopType).RegistryRouter(api) if err = r.Listen(":8080"); err != nil { glog.Warningf("server over: %v", err) } diff --git a/go.mod b/go.mod index cd33e2d..0e1dfc7 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.22.2 toolchain go1.22.3 require ( - gitea.timerzz.com/kedaya_haitao/common v0.0.0-20240614095701-e7fa9e7dd21a + gitea.timerzz.com/kedaya_haitao/common v0.0.0-20240616020231-a7d6c7c6661e github.com/gofiber/fiber/v3 v3.0.0-beta.2 github.com/golang/glog v1.2.1 github.com/samber/lo v1.39.0 @@ -41,6 +41,7 @@ require ( github.com/ericlagergren/polyval v0.0.0-20220411101811-e25bc10ba391 // indirect github.com/ericlagergren/siv v0.0.0-20220507050439-0b757b3aa5f1 // indirect github.com/ericlagergren/subtle v0.0.0-20220507045147-890d697da010 // indirect + github.com/expr-lang/expr v1.16.9 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/gaukas/godicttls v0.0.4 // indirect github.com/go-ole/go-ole v1.3.0 // indirect diff --git a/go.sum b/go.sum index 576acc9..89ca9ea 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,6 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -gitea.timerzz.com/kedaya_haitao/common v0.0.0-20240614095701-e7fa9e7dd21a h1:+O1PXfkfUQG5A830+q6YAg76jX663W8cMJxTCcGwjW0= -gitea.timerzz.com/kedaya_haitao/common v0.0.0-20240614095701-e7fa9e7dd21a/go.mod h1:MAmc7ooakm0o+4YshgKb+uE3Mfhex10s2RcJMiIYick= +gitea.timerzz.com/kedaya_haitao/common v0.0.0-20240616020231-a7d6c7c6661e h1:WCJfnIO44DZmkMdEoyZLMUSG68btBZ6FzVAnXe68Spg= +gitea.timerzz.com/kedaya_haitao/common v0.0.0-20240616020231-a7d6c7c6661e/go.mod h1:uV586p6Z8LIq3I8O/pXZv+jIkIwnwBjAz0D7KrhB9JM= github.com/3andne/restls-client-go v0.1.6 h1:tRx/YilqW7iHpgmEL4E1D8dAsuB0tFF3uvncS+B6I08= github.com/3andne/restls-client-go v0.1.6/go.mod h1:iEdTZNt9kzPIxjIGSMScUFSBrUH6bFRNg0BWlP4orEY= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= @@ -82,6 +82,8 @@ github.com/ericlagergren/siv v0.0.0-20220507050439-0b757b3aa5f1 h1:tlDMEdcPRQKBE github.com/ericlagergren/siv v0.0.0-20220507050439-0b757b3aa5f1/go.mod h1:4RfsapbGx2j/vU5xC/5/9qB3kn9Awp1YDiEnN43QrJ4= github.com/ericlagergren/subtle v0.0.0-20220507045147-890d697da010 h1:fuGucgPk5dN6wzfnxl3D0D3rVLw4v2SbBT9jb4VnxzA= github.com/ericlagergren/subtle v0.0.0-20220507045147-890d697da010/go.mod h1:JtBcj7sBuTTRupn7c2bFspMDIObMJsVK8TeUvpShPok= +github.com/expr-lang/expr v1.16.9 h1:WUAzmR0JNI9JCiF0/ewwHB1gmcGw5wW7nWt8gc6PpCI= +github.com/expr-lang/expr v1.16.9/go.mod h1:8/vRC7+7HBzESEqt5kKpYXxrxkr31SaO8r40VO/1IT4= github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= diff --git a/pkg/options/shop.go b/pkg/options/shop.go new file mode 100644 index 0000000..6a0c9f4 --- /dev/null +++ b/pkg/options/shop.go @@ -0,0 +1,6 @@ +package options + +const ( + ShopType_coach = "coach" + ShopType_outlet = "coach-outlet" +) diff --git a/server/spider.go b/server/spider.go new file mode 100644 index 0000000..63cc165 --- /dev/null +++ b/server/spider.go @@ -0,0 +1,83 @@ +package server + +import ( + "fmt" + "gitea.timerzz.com/kedaya_haitao/cn-coach-spider/spider" + productv1 "gitea.timerzz.com/kedaya_haitao/common/model/product" + "github.com/gofiber/fiber/v3" + "strconv" +) + +type SpiderSvc struct { + ctl *spider.Controller + shopType string +} + +func NewSpiderSvc(ctl *spider.Controller, shopType string) *SpiderSvc { + return &SpiderSvc{ + ctl: ctl, + shopType: shopType, + } +} + +func (s *SpiderSvc) RegistryRouter(r fiber.Router) { + r.Get(fmt.Sprintf("spider/cn/%s/global/calculate", s.shopType), s.GetGlobalCalculate) + r.Post(fmt.Sprintf("spider/cn/%s/global/calculate", s.shopType), s.UpsertGlobalCalculate) + r.Delete(fmt.Sprintf("spider/cn/%s/global/calculate/u/:id", s.shopType), s.DelGlobalCalculate) + r.Post(fmt.Sprintf("spider/cn/%s/calculate", s.shopType), s.UpsertCalculate) + r.Delete(fmt.Sprintf("spider/cn/%s/calculate/u/:id", s.shopType), s.DelCalculate) +} + +func (s *SpiderSvc) GetGlobalCalculate(ctx fiber.Ctx) error { + cals, err := s.ctl.GetGlobalCalculateProcess() + if err != nil { + return err + } + return ctx.JSON(cals) +} + +func (s *SpiderSvc) DelGlobalCalculate(ctx fiber.Ctx) error { + idStr := ctx.Params("id") + if idStr == "" { + return fiber.ErrBadRequest + } + id, _ := strconv.Atoi(idStr) + if err := s.ctl.DeleteGlobalCalculateProcess(uint(id)); err != nil { + return err + } + return nil +} + +func (s *SpiderSvc) UpsertGlobalCalculate(ctx fiber.Ctx) error { + var cs []productv1.CalculateProcess + if err := ctx.Bind().JSON(&cs); err != nil { + return err + } + if err := s.ctl.SaveGlobalCalculateProcess(cs); err != nil { + return err + } + return nil +} + +func (s *SpiderSvc) DelCalculate(ctx fiber.Ctx) error { + idStr := ctx.Params("id") + if idStr == "" { + return fiber.ErrBadRequest + } + id, _ := strconv.Atoi(idStr) + if err := s.ctl.DeleteCalculateProcess(uint(id)); err != nil { + return err + } + return nil +} + +func (s *SpiderSvc) UpsertCalculate(ctx fiber.Ctx) error { + var cs []productv1.CalculateProcess + if err := ctx.Bind().JSON(&cs); err != nil { + return err + } + if err := s.ctl.SaveCalculateProcess(cs); err != nil { + return err + } + return nil +} diff --git a/spider/calculate.go b/spider/calculate.go new file mode 100644 index 0000000..fedeb5f --- /dev/null +++ b/spider/calculate.go @@ -0,0 +1,115 @@ +package spider + +import ( + "errors" + productv1 "gitea.timerzz.com/kedaya_haitao/common/model/product" + "github.com/samber/lo" + "gorm.io/gorm" +) + +const ( + CNCoachPid = "cn-coach" + CNCoachOutletPid = "cn-coach-outlet" +) + +func (c *Controller) LoadCalculateProcess() { + c.db.Model(&productv1.CalculateProcess{}).Find(&c.calculates, "pid = ?", c.pid) +} + +func (c *Controller) GetGlobalCalculateProcess() (cs []productv1.CalculateProcess, err error) { + err = c.db.Model(&productv1.CalculateProcess{}).Find(&cs, "pid = ?", c.pid).Error + return +} + +// DeleteGlobalCalculateProcess 删除全局的计算规则 +func (c *Controller) DeleteGlobalCalculateProcess(id uint) error { + if _, idx, exist := lo.FindIndexOf(c.calculates, func(item productv1.CalculateProcess) bool { + return item.ID == id + }); exist { + if err := c.db.Model(&productv1.CalculateProcess{}).Delete(&c.calculates[idx], "id = ?", id).Error; err != nil { + return err + } + c.calculates = append(c.calculates[:idx], c.calculates[idx+1:]...) + go c.updateRate() + } + return nil +} + +// SaveGlobalCalculateProcess 更新全局的计算规则 +func (c *Controller) SaveGlobalCalculateProcess(cs []productv1.CalculateProcess) error { + if len(cs) == 0 { + return nil + } + cs = lo.Map(cs, func(item productv1.CalculateProcess, index int) productv1.CalculateProcess { + item.Pid = c.pid + return item + }) + return c.db.Transaction(func(tx *gorm.DB) error { + if err := c.db.Save(&cs).Error; err != nil { + return err + } + + if err := c.db.Model(&productv1.CalculateProcess{}).Find(&c.calculates, "pid = ?", c.pid).Error; err != nil { + return err + } + go c.updateRate() + return nil + }) +} + +// DeleteCalculateProcess 删除计算规则 +func (c *Controller) DeleteCalculateProcess(id uint) error { + return c.db.Transaction(func(tx *gorm.DB) error { + cal := productv1.CalculateProcess{} + if err := tx.Model(&productv1.CalculateProcess{}).Where("id = ?", id).First(&cal).Error; err != nil && !errors.As(err, &gorm.ErrRecordNotFound) { + return err + } else if errors.As(err, &gorm.ErrRecordNotFound) { + return nil + } + + if err := tx.Model(&productv1.CalculateProcess{}).Where("id = ?", cal.ID).Delete(&cal).Error; err != nil { + return err + } + var product productv1.Product + if err := tx.Preload("CalculateProcess").First(&product, "pid = ? AND website = ?", cal.Pid, c.website).Error; err != nil { + return err + } + product.CalCNY(append(product.CalculateProcess, c.calculates...)) + return tx.Model(&product).Where("pid = ? AND website = ?", product.Pid, c.website).Select("rate", "cal_mark", "cny_price").Updates(&product).Error + }) +} + +// SaveCalculateProcess 更新计算规则 +func (c *Controller) SaveCalculateProcess(cs []productv1.CalculateProcess) error { + if len(cs) == 0 { + return nil + } + + return c.db.Transaction(func(tx *gorm.DB) error { + if err := c.db.Save(&cs).Error; err != nil { + return err + } + + var product productv1.Product + if err := tx.Preload("CalculateProcess").First(&product, "pid = ? AND website = ?", cs[0].Pid, c.website).Error; err != nil { + return err + } + product.CalCNY(append(product.CalculateProcess, c.calculates...)) + return tx.Model(&product).Where("pid = ? AND website = ?", product.Pid, c.website).Select("rate", "cal_mark", "cny_price").Updates(&product).Error + }) +} + +func (c *Controller) updateRate() { + var results []*productv1.Product + c.db.Where("website = ?", c.website).FindInBatches(&results, 20, func(tx *gorm.DB, batch int) error { + for _, result := range results { + var calculate []productv1.CalculateProcess + c.db.Find(&calculate, "pid = ? AND website = ?", result.Pid, c.website) + result.CalCNY(append(calculate, c.calculates...)) + } + + // 保存对当前批记录的修改 + tx.Save(&results) + return nil + }) +} diff --git a/spider/controller.go b/spider/controller.go index b03910f..3c7fa2e 100644 --- a/spider/controller.go +++ b/spider/controller.go @@ -3,6 +3,7 @@ package spider import ( "context" "fmt" + "gitea.timerzz.com/kedaya_haitao/cn-coach-spider/pkg/options" "gitea.timerzz.com/kedaya_haitao/common/model/product" coach_client "gitea.timerzz.com/kedaya_haitao/common/pkg/coach-client" "github.com/golang/glog" @@ -13,19 +14,33 @@ import ( ) type Controller struct { - ctx context.Context - client *coach_client.CN - db *gorm.DB - interval time.Duration + ctx context.Context + client *coach_client.CN + db *gorm.DB + interval time.Duration + website productv1.Website + linkPrefix string + pid string + + calculates []productv1.CalculateProcess } -func NewController(client *coach_client.CN, db *gorm.DB, interval time.Duration) *Controller { +func NewController(client *coach_client.CN, db *gorm.DB, interval time.Duration, shopType string) *Controller { ctl := &Controller{ - client: client, - db: db, - interval: interval, + client: client, + db: db, + interval: interval, + website: productv1.WebSite_CN_Coach, + linkPrefix: "https://www.coach.com.cn/products/", + pid: CNCoachPid, + } + if shopType == options.ShopType_outlet { + ctl.website = productv1.WebSite_CN_Coach_Outlet + ctl.linkPrefix = "https://www.coach.com.cn/outlet/products/" + ctl.pid = CNCoachOutletPid } ctl.AutoMigrate() + ctl.LoadCalculateProcess() return ctl } @@ -34,6 +49,7 @@ func (c *Controller) AutoMigrate() { panic(err) } } + func (c *Controller) Run(ctx context.Context) { c.ctx = ctx ticker := time.NewTicker(c.interval) @@ -79,17 +95,24 @@ func (c *Controller) Crawl() error { func (c *Controller) saveRespData(list []coach_client.CNItem) error { var products = make([]productv1.Product, 0, len(list)) for _, item := range list { - products = append(products, productv1.Product{ - UpdatedAt: time.Now(), - Name: item.Title, - Pid: item.Code, - Link: fmt.Sprintf("https://www.coachoutlet.cn/products/%s", item.Code), - Image: item.Images[0].Imgs[0].Img, - Orderable: item.Stock > 0, - DiscPercent: 100 - int(item.DiscountRateMin*100), - CNYPrice: item.SkuMaxPrice, - Website: productv1.WebSite_CN_Coach, - }) + var savedProduct productv1.Product + c.db.Model(&savedProduct).Where("pid = ?", item.Code).Select("dw_price").Scan(&savedProduct) + p := productv1.Product{ + UpdatedAt: time.Now(), + Name: item.Title, + Pid: item.Code, + Link: fmt.Sprintf("%s%s", c.linkPrefix, item.Code), + Image: item.Images[0].Imgs[0].Img, + Orderable: item.Stock > 0, + DiscPercent: 100 - int(item.DiscountRateMin*100), + OriginalPrice: item.SkuMaxPrice, + Website: c.website, + DWPrice: savedProduct.DWPrice, + } + var calculate []productv1.CalculateProcess + c.db.Model(&productv1.CalculateProcess{}).Find(&calculate, "pid = ? AND website = ?", p.Pid, c.website) + p.CalCNY(append(calculate, c.calculates...)) + products = append(products, p) } // 去重 products = lo.UniqBy(products, func(p productv1.Product) string { @@ -97,6 +120,6 @@ func (c *Controller) saveRespData(list []coach_client.CNItem) error { }) return c.db.Clauses(clause.OnConflict{ Columns: []clause.Column{{Name: "pid"}}, - DoUpdates: clause.AssignmentColumns([]string{"name", "link", "orderable", "cny_price", "rate", "price_status", "disc_percent", "updated_at"}), + DoUpdates: clause.AssignmentColumns([]string{"name", "link", "orderable", "original_price", "rate", "price_status", "disc_percent", "updated_at"}), }).Create(products).Error }