kedaya_haitao/us-coach-spider
1
0
Fork 0
Code Issues 3 Pull Requests Actions Packages Projects Releases 1 Wiki Activity

feat 拉取ats

Browse Source
This commit is contained in:
timerzz 2024-11-21 14:36:31 +08:00
parent 6a41948924
commit 63633ede61
2 changed files with 147 additions and 60 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
server/spider.go
View File

@ -25,7 +25,10 @@ func NewSpiderSvc(ctl *spider.Controller, providerId v2.ProviderId) *SpiderSvc {
func (s *SpiderSvc) Registry(r fiber.Router) {
api := r.Group("/api/v2")
api.Post(fmt.Sprintf("provider/%s/pull", s.providerId), s.Pull)
api.Post(fmt.Sprintf("provider/%s/fetch/:id", s.providerId), s.FetchArticlePrice)
api.Post(fmt.Sprintf("provider/%s/price/fetch/:id", s.providerId), s.FetchArticlePrice)
api.Post(fmt.Sprintf("provider/%s/detail/fetch/:pid", s.providerId), s.FetchArticleDetail)
api.Post(fmt.Sprintf("provider/%s/ats/fetch/:pid", s.providerId), s.FetchArticleAts)
//r.Post("spider/us/coach-outlet/calculate", s.UpsertCalculate)
//r.Delete("spider/us/coach-outlet/calculate/u/:id", s.DelCalculate)
}
@ -48,6 +51,28 @@ func (s *SpiderSvc) FetchArticlePrice(ctx fiber.Ctx) error {
return ctx.JSON(web.NewResponse("ok"))
}
func (s *SpiderSvc) FetchArticleDetail(ctx fiber.Ctx) error {
pid := ctx.Params("pid")
if pid == "" {
return fmt.Errorf("pid is empty")
}
if err := s.ctl.FetchArticleDetail(ctx.Context(), pid); err != nil {
return err
}
return ctx.JSON(web.NewResponse("ok"))
}
func (s *SpiderSvc) FetchArticleAts(ctx fiber.Ctx) error {
pid := ctx.Params("pid")
if pid == "" {
return fmt.Errorf("pid is empty")
}
if err := s.ctl.FetchArticleAts(ctx.Context(), pid); err != nil {
return err
}
return ctx.JSON(web.NewResponse("ok"))
}
//func (s *SpiderSvc) DelCalculate(ctx fiber.Ctx) error {
// idStr := ctx.Params("id")
// if idStr == "" {

180
spider/controller.go
View File

@ -145,6 +145,48 @@ func (c *Controller) AutoMigrate() error {
return c.storage.ProviderArticle().AutoMigrate()
}
func (c *Controller) productsToArticles(products []coach_client.Product) (articles []v2.Article) {
for _, product := range products {
// 一个color是一个sku
for _, color := range product.Colors {
// 如果没找到，说明没有标准商品，创建一个
article := v2.Article{
Name: product.Name,
EnglishName: product.Name,
Pid: color.VgId,
Brand: v2.Brand_Coach,
Image: color.Media.Thumbnail.Src,
Providers: make([]v2.ProviderArticle, 0, 1),
}
pArticle := v2.ProviderArticle{
ProviderId: c.providerId,
Brand: v2.Brand_Coach,
Pid: color.VgId,
SkuID: color.VgId,
Available: color.Orderable,
Link: fmt.Sprintf("%s/%s", "https://www.coachoutlet.com", color.Url),
}
// 拿到现在的价格
price, _ := lo.Find(product.VariantsOnSale, func(item coach_client.Variant) bool {
return item.Id == color.VgId
})
// 计算成本
pArticle.Cost = utils.CalculateProviderPrice(
append(c.provider.CalculateProcess, pArticle.CalculateProcess...),
map[string]float64{
"originalPrice": price.Price.Sales.Value,
"freight": c.provider.Config.Freight,
"exchangeRate": c.provider.Config.ExchangeRate,
})
pArticle.HistoryPrice = append(pArticle.HistoryPrice, pArticle.Cost)
article.Providers = append(article.Providers, pArticle)
articles = append(articles, article)
}
}
return
}
// TODO长时间没更新的，需要更新
func (c *Controller) Crawl() {
glog.Infof("%s 开始抓取信息", time.Now())
// 开始拉取，修改状态
@ -161,7 +203,7 @@ func (c *Controller) Crawl() {
continue
}
totalPage = resp.PageData.TotalPages
c.saveProducts(resp.PageData.Products)
c.saveProducts(c.productsToArticles(resp.PageData.Products))
glog.Infof("第%d页数据保存完成", page)
break
}
@ -177,78 +219,98 @@ func (c *Controller) Crawl() {
}
// 对coach返回的数据进行处理保存
// TODO 如果供应商的价格比商品现在的最低价格低，那么需要通知计算商品的利润率
func (c *Controller) saveProducts(list []coach_client.Product) {
for _, resp := range list {
// 一个color是一个sku
for _, color := range resp.Colors {
article, err := c.storage.Article().Get(storage.NewGetArticleQuery().SetBrand(v2.Brand_Coach).SetPid(color.VgId))
if err != nil && !errors.As(err, &gorm.ErrRecordNotFound) {
glog.Errorf("获取商品信息失败: %v", err)
continue
}
// 如果没找到，说明没有标准商品，创建一个
if errors.As(err, &gorm.ErrRecordNotFound) {
article = v2.Article{
Name: resp.Name,
EnglishName: resp.Name,
Pid: color.VgId,
Brand: v2.Brand_Coach,
Image: color.Media.Thumbnail.Src,
Providers: make([]v2.ProviderArticle, 0, 1),
func (c *Controller) saveProducts(articles []v2.Article) {
for _, article := range articles {
oldArticle, err := c.storage.Article().Get(storage.NewGetArticleQuery().SetBrand(v2.Brand_Coach).SetPid(article.Pid))
if err != nil && !errors.As(err, &gorm.ErrRecordNotFound) {
glog.Errorf("获取商品信息失败: %v", err)
continue
}
// 如果已经存在了，那么不需要重新创建article
if err == nil {
// 查看现在有没有这个供应商的商品
oldProviderArticle, _, exist := lo.FindIndexOf(oldArticle.Providers, func(item v2.ProviderArticle) bool {
return item.ProviderId == c.providerId
})
// 创建供应商商品
if !exist {
oldArticle.Providers = append(oldArticle.Providers, article.Providers...)
// 保存商品信息
if err = c.storage.Article().Upsert(article); err != nil {
glog.Errorf("保存商品信息失败: %v", err)
continue
}
if err = c.storage.Article().Create(&article); err != nil {
glog.Errorf("创建商品信息失败: %v", err)
} else {
pArticle := article.Providers[0]
if oldProviderArticle.Cost.OriginalPrice != pArticle.Cost.OriginalPrice {
oldProviderArticle.HistoryPrice = append(oldProviderArticle.HistoryPrice, pArticle.Cost)
}
oldProviderArticle.Cost = pArticle.Cost
oldProviderArticle.Ast = pArticle.Ast
if err = c.storage.ProviderArticle().Upsert(oldProviderArticle); err != nil {
glog.Errorf("保存供应商商品信息失败: %v", err)
continue
}
}
pArticle, idx, exist := lo.FindIndexOf(article.Providers, func(item v2.ProviderArticle) bool {
return item.ProviderId == c.providerId
})
// 创建供应商上坪
if !exist {
pArticle = v2.ProviderArticle{
ProviderId: c.providerId,
Brand: v2.Brand_Coach,
Pid: color.VgId,
SkuID: color.VgId,
Available: color.Orderable,
Link: fmt.Sprintf("%s/%s", "https://www.coachoutlet.com", color.Url),
}
} else {
article.Providers = lo.DropByIndex(article.Providers, idx)
}
// 拿到现在的价格
price, _ := lo.Find(resp.VariantsOnSale, func(item coach_client.Variant) bool {
return item.Id == color.VgId
})
if originalPrice := price.Price.Sales.Value; originalPrice != pArticle.Cost.OriginalPrice {
// 价格发生了改变
pArticle.Cost = utils.CalculateProviderPrice(
append(c.provider.CalculateProcess, pArticle.CalculateProcess...),
map[string]float64{
"originalPrice": originalPrice,
"freight": c.provider.Config.Freight,
"exchangeRate": c.provider.Config.ExchangeRate,
})
pArticle.HistoryPrice = append(pArticle.HistoryPrice, pArticle.Cost)
}
article.Providers = append(article.Providers, pArticle)
} else {
// 如果article不存在，那么保存整个article
// 保存商品信息
if err = c.storage.Article().Upsert(article); err != nil {
glog.Errorf("保存商品信息失败: %v", err)
continue
}
if err = c.subscribeClient.Publish(c.ctx, utils.ProfitRate_Channel, strconv.Itoa(int(article.ID))); err != nil {
glog.Errorf("通知商品利润率失败: %v", err)
}
}
if err = c.subscribeClient.Publish(c.ctx, utils.ProfitRate_Channel, strconv.Itoa(int(article.ID))); err != nil {
glog.Errorf("通知商品利润率失败: %v", err)
}
}
}
func (c *Controller) FetchArticleDetail(ctx context.Context, pid string) error {
resp, err := c.client.RequestProductDetail(ctx, pid)
if err != nil {
return fmt.Errorf("请求商品信息失败: %v", err)
}
article := v2.Article{
Name: resp.Name,
EnglishName: resp.Name,
Pid: resp.Id,
Brand: v2.Brand_Coach,
Providers: []v2.ProviderArticle{
{
ProviderId: c.providerId,
Brand: v2.Brand_Coach,
Pid: pid,
SkuID: pid,
Available: resp.Inventory.Orderable,
Ast: resp.Inventory.Ats,
Link: fmt.Sprintf("%s/%s", "https://www.coachoutlet.com", resp.Url),
},
},
}
if len(resp.ImageGroups) > 0 {
article.Image = resp.ImageGroups[0].Images[0].Src
}
c.saveProducts([]v2.Article{article})
return nil
}
func (c *Controller)FetchArticleAts(ctx context.Context, pid string) error {
pArticle, err := c.storage.ProviderArticle().Get(storage.NewGetProviderArticleQuery().SetProviderId(c.providerId).SetPid(pid))
if err != nil {
return fmt.Errorf("获取商品信息失败: %v", err)
}
inv, err := c.client.RequestInventory(ctx, pid)
if err != nil {
return fmt.Errorf("请求商品库存失败: %v", err)
}
pArticle.Ast = inv.Ats
return c.storage.ProviderArticle().Update(pArticle, "ast")
}
// 更新某个商品的价格
func (c *Controller) FetchArticlePrice(ctx context.Context, id uint) error {
pArticle, err := c.storage.ProviderArticle().Get(storage.NewGetProviderArticleQuery().SetProviderId(c.providerId).SetID(id))