feat 支持coach和outlet
All checks were successful
Build image / build (push) Successful in 1m8s

This commit is contained in:
timerzz 2024-06-16 11:08:28 +08:00
parent 8eb26b08f6
commit d55cdfc0d5
7 changed files with 266 additions and 25 deletions

View File

@ -3,6 +3,7 @@ package main
import (
"context"
"gitea.timerzz.com/kedaya_haitao/cn-coach-spider/pkg/options"
"gitea.timerzz.com/kedaya_haitao/cn-coach-spider/server"
"gitea.timerzz.com/kedaya_haitao/cn-coach-spider/spider"
coach_client "gitea.timerzz.com/kedaya_haitao/common/pkg/coach-client"
"gitea.timerzz.com/kedaya_haitao/common/pkg/database"
@ -38,13 +39,21 @@ func main() {
Colorful: true,
},
)
shopType := os.Getenv("SHOP_TYPE")
if shopType != options.ShopType_coach && shopType != options.ShopType_outlet {
glog.Fatalf("SHOP_TYPE环境变量错误请设置SHOP_TYPE=coach或SHOP_TYPE=outlet")
}
// coach client
client, err := coach_client.CNClient()
var xMac = coach_client.XMac_Coach
if shopType == options.ShopType_outlet {
xMac = coach_client.XMac_Outlet
}
client, err := coach_client.CNClient(xMac)
if err != nil {
glog.Fatalf("初始化coach client失败%v", err)
}
spiderCtl := spider.NewController(client, db, cfg.Interval)
spiderCtl := spider.NewController(client, db, cfg.Interval, shopType)
go spiderCtl.Run(ctx)
@ -54,7 +63,9 @@ func main() {
r.Get("/health", func(ctx fiber.Ctx) error {
return ctx.SendString("ok")
})
api := r.Group("/api/v1")
server.NewSpiderSvc(spiderCtl, shopType).RegistryRouter(api)
if err = r.Listen(":8080"); err != nil {
glog.Warningf("server over: %v", err)
}

3
go.mod
View File

@ -5,7 +5,7 @@ go 1.22.2
toolchain go1.22.3
require (
gitea.timerzz.com/kedaya_haitao/common v0.0.0-20240614095701-e7fa9e7dd21a
gitea.timerzz.com/kedaya_haitao/common v0.0.0-20240616020231-a7d6c7c6661e
github.com/gofiber/fiber/v3 v3.0.0-beta.2
github.com/golang/glog v1.2.1
github.com/samber/lo v1.39.0
@ -41,6 +41,7 @@ require (
github.com/ericlagergren/polyval v0.0.0-20220411101811-e25bc10ba391 // indirect
github.com/ericlagergren/siv v0.0.0-20220507050439-0b757b3aa5f1 // indirect
github.com/ericlagergren/subtle v0.0.0-20220507045147-890d697da010 // indirect
github.com/expr-lang/expr v1.16.9 // indirect
github.com/fsnotify/fsnotify v1.7.0 // indirect
github.com/gaukas/godicttls v0.0.4 // indirect
github.com/go-ole/go-ole v1.3.0 // indirect

6
go.sum
View File

@ -1,6 +1,6 @@
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
gitea.timerzz.com/kedaya_haitao/common v0.0.0-20240614095701-e7fa9e7dd21a h1:+O1PXfkfUQG5A830+q6YAg76jX663W8cMJxTCcGwjW0=
gitea.timerzz.com/kedaya_haitao/common v0.0.0-20240614095701-e7fa9e7dd21a/go.mod h1:MAmc7ooakm0o+4YshgKb+uE3Mfhex10s2RcJMiIYick=
gitea.timerzz.com/kedaya_haitao/common v0.0.0-20240616020231-a7d6c7c6661e h1:WCJfnIO44DZmkMdEoyZLMUSG68btBZ6FzVAnXe68Spg=
gitea.timerzz.com/kedaya_haitao/common v0.0.0-20240616020231-a7d6c7c6661e/go.mod h1:uV586p6Z8LIq3I8O/pXZv+jIkIwnwBjAz0D7KrhB9JM=
github.com/3andne/restls-client-go v0.1.6 h1:tRx/YilqW7iHpgmEL4E1D8dAsuB0tFF3uvncS+B6I08=
github.com/3andne/restls-client-go v0.1.6/go.mod h1:iEdTZNt9kzPIxjIGSMScUFSBrUH6bFRNg0BWlP4orEY=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
@ -82,6 +82,8 @@ github.com/ericlagergren/siv v0.0.0-20220507050439-0b757b3aa5f1 h1:tlDMEdcPRQKBE
github.com/ericlagergren/siv v0.0.0-20220507050439-0b757b3aa5f1/go.mod h1:4RfsapbGx2j/vU5xC/5/9qB3kn9Awp1YDiEnN43QrJ4=
github.com/ericlagergren/subtle v0.0.0-20220507045147-890d697da010 h1:fuGucgPk5dN6wzfnxl3D0D3rVLw4v2SbBT9jb4VnxzA=
github.com/ericlagergren/subtle v0.0.0-20220507045147-890d697da010/go.mod h1:JtBcj7sBuTTRupn7c2bFspMDIObMJsVK8TeUvpShPok=
github.com/expr-lang/expr v1.16.9 h1:WUAzmR0JNI9JCiF0/ewwHB1gmcGw5wW7nWt8gc6PpCI=
github.com/expr-lang/expr v1.16.9/go.mod h1:8/vRC7+7HBzESEqt5kKpYXxrxkr31SaO8r40VO/1IT4=
github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU=
github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw=
github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA=

6
pkg/options/shop.go Normal file
View File

@ -0,0 +1,6 @@
package options
const (
ShopType_coach = "coach"
ShopType_outlet = "coach-outlet"
)

83
server/spider.go Normal file
View File

@ -0,0 +1,83 @@
package server
import (
"fmt"
"gitea.timerzz.com/kedaya_haitao/cn-coach-spider/spider"
productv1 "gitea.timerzz.com/kedaya_haitao/common/model/product"
"github.com/gofiber/fiber/v3"
"strconv"
)
type SpiderSvc struct {
ctl *spider.Controller
shopType string
}
func NewSpiderSvc(ctl *spider.Controller, shopType string) *SpiderSvc {
return &SpiderSvc{
ctl: ctl,
shopType: shopType,
}
}
func (s *SpiderSvc) RegistryRouter(r fiber.Router) {
r.Get(fmt.Sprintf("spider/cn/%s/global/calculate", s.shopType), s.GetGlobalCalculate)
r.Post(fmt.Sprintf("spider/cn/%s/global/calculate", s.shopType), s.UpsertGlobalCalculate)
r.Delete(fmt.Sprintf("spider/cn/%s/global/calculate/u/:id", s.shopType), s.DelGlobalCalculate)
r.Post(fmt.Sprintf("spider/cn/%s/calculate", s.shopType), s.UpsertCalculate)
r.Delete(fmt.Sprintf("spider/cn/%s/calculate/u/:id", s.shopType), s.DelCalculate)
}
func (s *SpiderSvc) GetGlobalCalculate(ctx fiber.Ctx) error {
cals, err := s.ctl.GetGlobalCalculateProcess()
if err != nil {
return err
}
return ctx.JSON(cals)
}
func (s *SpiderSvc) DelGlobalCalculate(ctx fiber.Ctx) error {
idStr := ctx.Params("id")
if idStr == "" {
return fiber.ErrBadRequest
}
id, _ := strconv.Atoi(idStr)
if err := s.ctl.DeleteGlobalCalculateProcess(uint(id)); err != nil {
return err
}
return nil
}
func (s *SpiderSvc) UpsertGlobalCalculate(ctx fiber.Ctx) error {
var cs []productv1.CalculateProcess
if err := ctx.Bind().JSON(&cs); err != nil {
return err
}
if err := s.ctl.SaveGlobalCalculateProcess(cs); err != nil {
return err
}
return nil
}
func (s *SpiderSvc) DelCalculate(ctx fiber.Ctx) error {
idStr := ctx.Params("id")
if idStr == "" {
return fiber.ErrBadRequest
}
id, _ := strconv.Atoi(idStr)
if err := s.ctl.DeleteCalculateProcess(uint(id)); err != nil {
return err
}
return nil
}
func (s *SpiderSvc) UpsertCalculate(ctx fiber.Ctx) error {
var cs []productv1.CalculateProcess
if err := ctx.Bind().JSON(&cs); err != nil {
return err
}
if err := s.ctl.SaveCalculateProcess(cs); err != nil {
return err
}
return nil
}

115
spider/calculate.go Normal file
View File

@ -0,0 +1,115 @@
package spider
import (
"errors"
productv1 "gitea.timerzz.com/kedaya_haitao/common/model/product"
"github.com/samber/lo"
"gorm.io/gorm"
)
const (
CNCoachPid = "cn-coach"
CNCoachOutletPid = "cn-coach-outlet"
)
func (c *Controller) LoadCalculateProcess() {
c.db.Model(&productv1.CalculateProcess{}).Find(&c.calculates, "pid = ?", c.pid)
}
func (c *Controller) GetGlobalCalculateProcess() (cs []productv1.CalculateProcess, err error) {
err = c.db.Model(&productv1.CalculateProcess{}).Find(&cs, "pid = ?", c.pid).Error
return
}
// DeleteGlobalCalculateProcess 删除全局的计算规则
func (c *Controller) DeleteGlobalCalculateProcess(id uint) error {
if _, idx, exist := lo.FindIndexOf(c.calculates, func(item productv1.CalculateProcess) bool {
return item.ID == id
}); exist {
if err := c.db.Model(&productv1.CalculateProcess{}).Delete(&c.calculates[idx], "id = ?", id).Error; err != nil {
return err
}
c.calculates = append(c.calculates[:idx], c.calculates[idx+1:]...)
go c.updateRate()
}
return nil
}
// SaveGlobalCalculateProcess 更新全局的计算规则
func (c *Controller) SaveGlobalCalculateProcess(cs []productv1.CalculateProcess) error {
if len(cs) == 0 {
return nil
}
cs = lo.Map(cs, func(item productv1.CalculateProcess, index int) productv1.CalculateProcess {
item.Pid = c.pid
return item
})
return c.db.Transaction(func(tx *gorm.DB) error {
if err := c.db.Save(&cs).Error; err != nil {
return err
}
if err := c.db.Model(&productv1.CalculateProcess{}).Find(&c.calculates, "pid = ?", c.pid).Error; err != nil {
return err
}
go c.updateRate()
return nil
})
}
// DeleteCalculateProcess 删除计算规则
func (c *Controller) DeleteCalculateProcess(id uint) error {
return c.db.Transaction(func(tx *gorm.DB) error {
cal := productv1.CalculateProcess{}
if err := tx.Model(&productv1.CalculateProcess{}).Where("id = ?", id).First(&cal).Error; err != nil && !errors.As(err, &gorm.ErrRecordNotFound) {
return err
} else if errors.As(err, &gorm.ErrRecordNotFound) {
return nil
}
if err := tx.Model(&productv1.CalculateProcess{}).Where("id = ?", cal.ID).Delete(&cal).Error; err != nil {
return err
}
var product productv1.Product
if err := tx.Preload("CalculateProcess").First(&product, "pid = ? AND website = ?", cal.Pid, c.website).Error; err != nil {
return err
}
product.CalCNY(append(product.CalculateProcess, c.calculates...))
return tx.Model(&product).Where("pid = ? AND website = ?", product.Pid, c.website).Select("rate", "cal_mark", "cny_price").Updates(&product).Error
})
}
// SaveCalculateProcess 更新计算规则
func (c *Controller) SaveCalculateProcess(cs []productv1.CalculateProcess) error {
if len(cs) == 0 {
return nil
}
return c.db.Transaction(func(tx *gorm.DB) error {
if err := c.db.Save(&cs).Error; err != nil {
return err
}
var product productv1.Product
if err := tx.Preload("CalculateProcess").First(&product, "pid = ? AND website = ?", cs[0].Pid, c.website).Error; err != nil {
return err
}
product.CalCNY(append(product.CalculateProcess, c.calculates...))
return tx.Model(&product).Where("pid = ? AND website = ?", product.Pid, c.website).Select("rate", "cal_mark", "cny_price").Updates(&product).Error
})
}
func (c *Controller) updateRate() {
var results []*productv1.Product
c.db.Where("website = ?", c.website).FindInBatches(&results, 20, func(tx *gorm.DB, batch int) error {
for _, result := range results {
var calculate []productv1.CalculateProcess
c.db.Find(&calculate, "pid = ? AND website = ?", result.Pid, c.website)
result.CalCNY(append(calculate, c.calculates...))
}
// 保存对当前批记录的修改
tx.Save(&results)
return nil
})
}

View File

@ -3,6 +3,7 @@ package spider
import (
"context"
"fmt"
"gitea.timerzz.com/kedaya_haitao/cn-coach-spider/pkg/options"
"gitea.timerzz.com/kedaya_haitao/common/model/product"
coach_client "gitea.timerzz.com/kedaya_haitao/common/pkg/coach-client"
"github.com/golang/glog"
@ -17,15 +18,29 @@ type Controller struct {
client *coach_client.CN
db *gorm.DB
interval time.Duration
website productv1.Website
linkPrefix string
pid string
calculates []productv1.CalculateProcess
}
func NewController(client *coach_client.CN, db *gorm.DB, interval time.Duration) *Controller {
func NewController(client *coach_client.CN, db *gorm.DB, interval time.Duration, shopType string) *Controller {
ctl := &Controller{
client: client,
db: db,
interval: interval,
website: productv1.WebSite_CN_Coach,
linkPrefix: "https://www.coach.com.cn/products/",
pid: CNCoachPid,
}
if shopType == options.ShopType_outlet {
ctl.website = productv1.WebSite_CN_Coach_Outlet
ctl.linkPrefix = "https://www.coach.com.cn/outlet/products/"
ctl.pid = CNCoachOutletPid
}
ctl.AutoMigrate()
ctl.LoadCalculateProcess()
return ctl
}
@ -34,6 +49,7 @@ func (c *Controller) AutoMigrate() {
panic(err)
}
}
func (c *Controller) Run(ctx context.Context) {
c.ctx = ctx
ticker := time.NewTicker(c.interval)
@ -79,17 +95,24 @@ func (c *Controller) Crawl() error {
func (c *Controller) saveRespData(list []coach_client.CNItem) error {
var products = make([]productv1.Product, 0, len(list))
for _, item := range list {
products = append(products, productv1.Product{
var savedProduct productv1.Product
c.db.Model(&savedProduct).Where("pid = ?", item.Code).Select("dw_price").Scan(&savedProduct)
p := productv1.Product{
UpdatedAt: time.Now(),
Name: item.Title,
Pid: item.Code,
Link: fmt.Sprintf("https://www.coachoutlet.cn/products/%s", item.Code),
Link: fmt.Sprintf("%s%s", c.linkPrefix, item.Code),
Image: item.Images[0].Imgs[0].Img,
Orderable: item.Stock > 0,
DiscPercent: 100 - int(item.DiscountRateMin*100),
CNYPrice: item.SkuMaxPrice,
Website: productv1.WebSite_CN_Coach,
})
OriginalPrice: item.SkuMaxPrice,
Website: c.website,
DWPrice: savedProduct.DWPrice,
}
var calculate []productv1.CalculateProcess
c.db.Model(&productv1.CalculateProcess{}).Find(&calculate, "pid = ? AND website = ?", p.Pid, c.website)
p.CalCNY(append(calculate, c.calculates...))
products = append(products, p)
}
// 去重
products = lo.UniqBy(products, func(p productv1.Product) string {
@ -97,6 +120,6 @@ func (c *Controller) saveRespData(list []coach_client.CNItem) error {
})
return c.db.Clauses(clause.OnConflict{
Columns: []clause.Column{{Name: "pid"}},
DoUpdates: clause.AssignmentColumns([]string{"name", "link", "orderable", "cny_price", "rate", "price_status", "disc_percent", "updated_at"}),
DoUpdates: clause.AssignmentColumns([]string{"name", "link", "orderable", "original_price", "rate", "price_status", "disc_percent", "updated_at"}),
}).Create(products).Error
}