feat 支持coach和outlet

2024-06-16 11:08:28 +08:00 · 2024-06-16 11:08:28 +08:00 · d55cdfc0d5
commit d55cdfc0d5
parent 8eb26b08f6
7 changed files with 266 additions and 25 deletions
--- a/cmd/main.go
+++ b/cmd/main.go
@ -3,6 +3,7 @@ package main
 import (
 	"context"
 	"gitea.timerzz.com/kedaya_haitao/cn-coach-spider/pkg/options"
+	"gitea.timerzz.com/kedaya_haitao/cn-coach-spider/server"
 	"gitea.timerzz.com/kedaya_haitao/cn-coach-spider/spider"
 	coach_client "gitea.timerzz.com/kedaya_haitao/common/pkg/coach-client"
 	"gitea.timerzz.com/kedaya_haitao/common/pkg/database"
@ -38,13 +39,21 @@ func main() {
 			Colorful:                  true,
 		},
 	)
+	shopType := os.Getenv("SHOP_TYPE")
+	if shopType != options.ShopType_coach && shopType != options.ShopType_outlet {
+		glog.Fatalf("SHOP_TYPE环境变量错误，请设置SHOP_TYPE=coach或SHOP_TYPE=outlet")
+	}
 	// coach client
-	client, err := coach_client.CNClient()
+	var xMac = coach_client.XMac_Coach
+	if shopType == options.ShopType_outlet {
+		xMac = coach_client.XMac_Outlet
+	}
+	client, err := coach_client.CNClient(xMac)
 	if err != nil {
 		glog.Fatalf("初始化coach client失败：%v", err)
 	}

-	spiderCtl := spider.NewController(client, db, cfg.Interval)
+	spiderCtl := spider.NewController(client, db, cfg.Interval, shopType)

 	go spiderCtl.Run(ctx)

@ -54,7 +63,9 @@ func main() {
 	r.Get("/health", func(ctx fiber.Ctx) error {
 		return ctx.SendString("ok")
 	})
+	api := r.Group("/api/v1")

+	server.NewSpiderSvc(spiderCtl, shopType).RegistryRouter(api)
 	if err = r.Listen(":8080"); err != nil {
 		glog.Warningf("server over: %v", err)
 	}
--- a/go.mod
+++ b/go.mod
@ -5,7 +5,7 @@ go 1.22.2
 toolchain go1.22.3

 require (
-	gitea.timerzz.com/kedaya_haitao/common v0.0.0-20240614095701-e7fa9e7dd21a
+	gitea.timerzz.com/kedaya_haitao/common v0.0.0-20240616020231-a7d6c7c6661e
 	github.com/gofiber/fiber/v3 v3.0.0-beta.2
 	github.com/golang/glog v1.2.1
 	github.com/samber/lo v1.39.0
@ -41,6 +41,7 @@ require (
 	github.com/ericlagergren/polyval v0.0.0-20220411101811-e25bc10ba391 // indirect
 	github.com/ericlagergren/siv v0.0.0-20220507050439-0b757b3aa5f1 // indirect
 	github.com/ericlagergren/subtle v0.0.0-20220507045147-890d697da010 // indirect
+	github.com/expr-lang/expr v1.16.9 // indirect
 	github.com/fsnotify/fsnotify v1.7.0 // indirect
 	github.com/gaukas/godicttls v0.0.4 // indirect
 	github.com/go-ole/go-ole v1.3.0 // indirect
--- a/go.sum
+++ b/go.sum
@ -1,6 +1,6 @@
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
-gitea.timerzz.com/kedaya_haitao/common v0.0.0-20240614095701-e7fa9e7dd21a h1:+O1PXfkfUQG5A830+q6YAg76jX663W8cMJxTCcGwjW0=
-gitea.timerzz.com/kedaya_haitao/common v0.0.0-20240614095701-e7fa9e7dd21a/go.mod h1:MAmc7ooakm0o+4YshgKb+uE3Mfhex10s2RcJMiIYick=
+gitea.timerzz.com/kedaya_haitao/common v0.0.0-20240616020231-a7d6c7c6661e h1:WCJfnIO44DZmkMdEoyZLMUSG68btBZ6FzVAnXe68Spg=
+gitea.timerzz.com/kedaya_haitao/common v0.0.0-20240616020231-a7d6c7c6661e/go.mod h1:uV586p6Z8LIq3I8O/pXZv+jIkIwnwBjAz0D7KrhB9JM=
 github.com/3andne/restls-client-go v0.1.6 h1:tRx/YilqW7iHpgmEL4E1D8dAsuB0tFF3uvncS+B6I08=
 github.com/3andne/restls-client-go v0.1.6/go.mod h1:iEdTZNt9kzPIxjIGSMScUFSBrUH6bFRNg0BWlP4orEY=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
@ -82,6 +82,8 @@ github.com/ericlagergren/siv v0.0.0-20220507050439-0b757b3aa5f1 h1:tlDMEdcPRQKBE
 github.com/ericlagergren/siv v0.0.0-20220507050439-0b757b3aa5f1/go.mod h1:4RfsapbGx2j/vU5xC/5/9qB3kn9Awp1YDiEnN43QrJ4=
 github.com/ericlagergren/subtle v0.0.0-20220507045147-890d697da010 h1:fuGucgPk5dN6wzfnxl3D0D3rVLw4v2SbBT9jb4VnxzA=
 github.com/ericlagergren/subtle v0.0.0-20220507045147-890d697da010/go.mod h1:JtBcj7sBuTTRupn7c2bFspMDIObMJsVK8TeUvpShPok=
+github.com/expr-lang/expr v1.16.9 h1:WUAzmR0JNI9JCiF0/ewwHB1gmcGw5wW7nWt8gc6PpCI=
+github.com/expr-lang/expr v1.16.9/go.mod h1:8/vRC7+7HBzESEqt5kKpYXxrxkr31SaO8r40VO/1IT4=
 github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU=
 github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw=
 github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA=
--- a/pkg/options/shop.go
+++ b/pkg/options/shop.go
@ -0,0 +1,6 @@
+package options
+
+const (
+	ShopType_coach  = "coach"
+	ShopType_outlet = "coach-outlet"
+)
--- a/server/spider.go
+++ b/server/spider.go
@ -0,0 +1,83 @@
+package server
+
+import (
+	"fmt"
+	"gitea.timerzz.com/kedaya_haitao/cn-coach-spider/spider"
+	productv1 "gitea.timerzz.com/kedaya_haitao/common/model/product"
+	"github.com/gofiber/fiber/v3"
+	"strconv"
+)
+
+type SpiderSvc struct {
+	ctl      *spider.Controller
+	shopType string
+}
+
+func NewSpiderSvc(ctl *spider.Controller, shopType string) *SpiderSvc {
+	return &SpiderSvc{
+		ctl:      ctl,
+		shopType: shopType,
+	}
+}
+
+func (s *SpiderSvc) RegistryRouter(r fiber.Router) {
+	r.Get(fmt.Sprintf("spider/cn/%s/global/calculate", s.shopType), s.GetGlobalCalculate)
+	r.Post(fmt.Sprintf("spider/cn/%s/global/calculate", s.shopType), s.UpsertGlobalCalculate)
+	r.Delete(fmt.Sprintf("spider/cn/%s/global/calculate/u/:id", s.shopType), s.DelGlobalCalculate)
+	r.Post(fmt.Sprintf("spider/cn/%s/calculate", s.shopType), s.UpsertCalculate)
+	r.Delete(fmt.Sprintf("spider/cn/%s/calculate/u/:id", s.shopType), s.DelCalculate)
+}
+
+func (s *SpiderSvc) GetGlobalCalculate(ctx fiber.Ctx) error {
+	cals, err := s.ctl.GetGlobalCalculateProcess()
+	if err != nil {
+		return err
+	}
+	return ctx.JSON(cals)
+}
+
+func (s *SpiderSvc) DelGlobalCalculate(ctx fiber.Ctx) error {
+	idStr := ctx.Params("id")
+	if idStr == "" {
+		return fiber.ErrBadRequest
+	}
+	id, _ := strconv.Atoi(idStr)
+	if err := s.ctl.DeleteGlobalCalculateProcess(uint(id)); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (s *SpiderSvc) UpsertGlobalCalculate(ctx fiber.Ctx) error {
+	var cs []productv1.CalculateProcess
+	if err := ctx.Bind().JSON(&cs); err != nil {
+		return err
+	}
+	if err := s.ctl.SaveGlobalCalculateProcess(cs); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (s *SpiderSvc) DelCalculate(ctx fiber.Ctx) error {
+	idStr := ctx.Params("id")
+	if idStr == "" {
+		return fiber.ErrBadRequest
+	}
+	id, _ := strconv.Atoi(idStr)
+	if err := s.ctl.DeleteCalculateProcess(uint(id)); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (s *SpiderSvc) UpsertCalculate(ctx fiber.Ctx) error {
+	var cs []productv1.CalculateProcess
+	if err := ctx.Bind().JSON(&cs); err != nil {
+		return err
+	}
+	if err := s.ctl.SaveCalculateProcess(cs); err != nil {
+		return err
+	}
+	return nil
+}
--- a/spider/calculate.go
+++ b/spider/calculate.go
@ -0,0 +1,115 @@
+package spider
+
+import (
+	"errors"
+	productv1 "gitea.timerzz.com/kedaya_haitao/common/model/product"
+	"github.com/samber/lo"
+	"gorm.io/gorm"
+)
+
+const (
+	CNCoachPid       = "cn-coach"
+	CNCoachOutletPid = "cn-coach-outlet"
+)
+
+func (c *Controller) LoadCalculateProcess() {
+	c.db.Model(&productv1.CalculateProcess{}).Find(&c.calculates, "pid = ?", c.pid)
+}
+
+func (c *Controller) GetGlobalCalculateProcess() (cs []productv1.CalculateProcess, err error) {
+	err = c.db.Model(&productv1.CalculateProcess{}).Find(&cs, "pid = ?", c.pid).Error
+	return
+}
+
+// DeleteGlobalCalculateProcess 删除全局的计算规则
+func (c *Controller) DeleteGlobalCalculateProcess(id uint) error {
+	if _, idx, exist := lo.FindIndexOf(c.calculates, func(item productv1.CalculateProcess) bool {
+		return item.ID == id
+	}); exist {
+		if err := c.db.Model(&productv1.CalculateProcess{}).Delete(&c.calculates[idx], "id = ?", id).Error; err != nil {
+			return err
+		}
+		c.calculates = append(c.calculates[:idx], c.calculates[idx+1:]...)
+		go c.updateRate()
+	}
+	return nil
+}
+
+// SaveGlobalCalculateProcess 更新全局的计算规则
+func (c *Controller) SaveGlobalCalculateProcess(cs []productv1.CalculateProcess) error {
+	if len(cs) == 0 {
+		return nil
+	}
+	cs = lo.Map(cs, func(item productv1.CalculateProcess, index int) productv1.CalculateProcess {
+		item.Pid = c.pid
+		return item
+	})
+	return c.db.Transaction(func(tx *gorm.DB) error {
+		if err := c.db.Save(&cs).Error; err != nil {
+			return err
+		}
+
+		if err := c.db.Model(&productv1.CalculateProcess{}).Find(&c.calculates, "pid = ?", c.pid).Error; err != nil {
+			return err
+		}
+		go c.updateRate()
+		return nil
+	})
+}
+
+// DeleteCalculateProcess 删除计算规则
+func (c *Controller) DeleteCalculateProcess(id uint) error {
+	return c.db.Transaction(func(tx *gorm.DB) error {
+		cal := productv1.CalculateProcess{}
+		if err := tx.Model(&productv1.CalculateProcess{}).Where("id = ?", id).First(&cal).Error; err != nil && !errors.As(err, &gorm.ErrRecordNotFound) {
+			return err
+		} else if errors.As(err, &gorm.ErrRecordNotFound) {
+			return nil
+		}
+
+		if err := tx.Model(&productv1.CalculateProcess{}).Where("id = ?", cal.ID).Delete(&cal).Error; err != nil {
+			return err
+		}
+		var product productv1.Product
+		if err := tx.Preload("CalculateProcess").First(&product, "pid = ? AND website = ?", cal.Pid, c.website).Error; err != nil {
+			return err
+		}
+		product.CalCNY(append(product.CalculateProcess, c.calculates...))
+		return tx.Model(&product).Where("pid = ? AND website = ?", product.Pid, c.website).Select("rate", "cal_mark", "cny_price").Updates(&product).Error
+	})
+}
+
+// SaveCalculateProcess 更新计算规则
+func (c *Controller) SaveCalculateProcess(cs []productv1.CalculateProcess) error {
+	if len(cs) == 0 {
+		return nil
+	}
+
+	return c.db.Transaction(func(tx *gorm.DB) error {
+		if err := c.db.Save(&cs).Error; err != nil {
+			return err
+		}
+
+		var product productv1.Product
+		if err := tx.Preload("CalculateProcess").First(&product, "pid = ? AND website = ?", cs[0].Pid, c.website).Error; err != nil {
+			return err
+		}
+		product.CalCNY(append(product.CalculateProcess, c.calculates...))
+		return tx.Model(&product).Where("pid = ? AND website = ?", product.Pid, c.website).Select("rate", "cal_mark", "cny_price").Updates(&product).Error
+	})
+}
+
+func (c *Controller) updateRate() {
+	var results []*productv1.Product
+	c.db.Where("website = ?", c.website).FindInBatches(&results, 20, func(tx *gorm.DB, batch int) error {
+		for _, result := range results {
+			var calculate []productv1.CalculateProcess
+			c.db.Find(&calculate, "pid = ? AND website = ?", result.Pid, c.website)
+			result.CalCNY(append(calculate, c.calculates...))
+		}
+
+		// 保存对当前批记录的修改
+		tx.Save(&results)
+		return nil
+	})
+}
--- a/spider/controller.go
+++ b/spider/controller.go
@ -3,6 +3,7 @@ package spider
 import (
 	"context"
 	"fmt"
+	"gitea.timerzz.com/kedaya_haitao/cn-coach-spider/pkg/options"
 	"gitea.timerzz.com/kedaya_haitao/common/model/product"
 	coach_client "gitea.timerzz.com/kedaya_haitao/common/pkg/coach-client"
 	"github.com/golang/glog"
@ -13,19 +14,33 @@ import (
 )

 type Controller struct {
-	ctx      context.Context
-	client   *coach_client.CN
-	db       *gorm.DB
-	interval time.Duration
+	ctx        context.Context
+	client     *coach_client.CN
+	db         *gorm.DB
+	interval   time.Duration
+	website    productv1.Website
+	linkPrefix string
+	pid        string
+
+	calculates []productv1.CalculateProcess
 }

-func NewController(client *coach_client.CN, db *gorm.DB, interval time.Duration) *Controller {
+func NewController(client *coach_client.CN, db *gorm.DB, interval time.Duration, shopType string) *Controller {
 	ctl := &Controller{
-		client:   client,
-		db:       db,
-		interval: interval,
+		client:     client,
+		db:         db,
+		interval:   interval,
+		website:    productv1.WebSite_CN_Coach,
+		linkPrefix: "https://www.coach.com.cn/products/",
+		pid:        CNCoachPid,
+	}
+	if shopType == options.ShopType_outlet {
+		ctl.website = productv1.WebSite_CN_Coach_Outlet
+		ctl.linkPrefix = "https://www.coach.com.cn/outlet/products/"
+		ctl.pid = CNCoachOutletPid
 	}
 	ctl.AutoMigrate()
+	ctl.LoadCalculateProcess()
 	return ctl
 }

@ -34,6 +49,7 @@ func (c *Controller) AutoMigrate() {
 		panic(err)
 	}
 }
+
 func (c *Controller) Run(ctx context.Context) {
 	c.ctx = ctx
 	ticker := time.NewTicker(c.interval)
@ -79,17 +95,24 @@ func (c *Controller) Crawl() error {
 func (c *Controller) saveRespData(list []coach_client.CNItem) error {
 	var products = make([]productv1.Product, 0, len(list))
 	for _, item := range list {
-		products = append(products, productv1.Product{
-			UpdatedAt:   time.Now(),
-			Name:        item.Title,
-			Pid:         item.Code,
-			Link:        fmt.Sprintf("https://www.coachoutlet.cn/products/%s", item.Code),
-			Image:       item.Images[0].Imgs[0].Img,
-			Orderable:   item.Stock > 0,
-			DiscPercent: 100 - int(item.DiscountRateMin*100),
-			CNYPrice:    item.SkuMaxPrice,
-			Website:     productv1.WebSite_CN_Coach,
-		})
+		var savedProduct productv1.Product
+		c.db.Model(&savedProduct).Where("pid = ?", item.Code).Select("dw_price").Scan(&savedProduct)
+		p := productv1.Product{
+			UpdatedAt:     time.Now(),
+			Name:          item.Title,
+			Pid:           item.Code,
+			Link:          fmt.Sprintf("%s%s", c.linkPrefix, item.Code),
+			Image:         item.Images[0].Imgs[0].Img,
+			Orderable:     item.Stock > 0,
+			DiscPercent:   100 - int(item.DiscountRateMin*100),
+			OriginalPrice: item.SkuMaxPrice,
+			Website:       c.website,
+			DWPrice:       savedProduct.DWPrice,
+		}
+		var calculate []productv1.CalculateProcess
+		c.db.Model(&productv1.CalculateProcess{}).Find(&calculate, "pid = ? AND website = ?", p.Pid, c.website)
+		p.CalCNY(append(calculate, c.calculates...))
+		products = append(products, p)
 	}
 	// 去重
 	products = lo.UniqBy(products, func(p productv1.Product) string {
@ -97,6 +120,6 @@ func (c *Controller) saveRespData(list []coach_client.CNItem) error {
 	})
 	return c.db.Clauses(clause.OnConflict{
 		Columns:   []clause.Column{{Name: "pid"}},
-		DoUpdates: clause.AssignmentColumns([]string{"name", "link", "orderable", "cny_price", "rate", "price_status", "disc_percent", "updated_at"}),
+		DoUpdates: clause.AssignmentColumns([]string{"name", "link", "orderable", "original_price", "rate", "price_status", "disc_percent", "updated_at"}),
 	}).Create(products).Error
 }