feat 拉取ats

server/spider.go

@@ -25,7 +25,10 @@ func NewSpiderSvc(ctl *spider.Controller, providerId v2.ProviderId) *SpiderSvc {
 func (s *SpiderSvc) Registry(r fiber.Router) {
 	api := r.Group("/api/v2")
 	api.Post(fmt.Sprintf("provider/%s/pull", s.providerId), s.Pull)
-	api.Post(fmt.Sprintf("provider/%s/fetch/:id", s.providerId), s.FetchArticlePrice)
+	api.Post(fmt.Sprintf("provider/%s/price/fetch/:id", s.providerId), s.FetchArticlePrice)
+	api.Post(fmt.Sprintf("provider/%s/detail/fetch/:pid", s.providerId), s.FetchArticleDetail)
+	api.Post(fmt.Sprintf("provider/%s/ats/fetch/:pid", s.providerId), s.FetchArticleAts)
+
 	//r.Post("spider/us/coach-outlet/calculate", s.UpsertCalculate)
 	//r.Delete("spider/us/coach-outlet/calculate/u/:id", s.DelCalculate)
 }
@@ -48,6 +51,28 @@ func (s *SpiderSvc) FetchArticlePrice(ctx fiber.Ctx) error {
 	return ctx.JSON(web.NewResponse("ok"))
 }
 
+func (s *SpiderSvc) FetchArticleDetail(ctx fiber.Ctx) error {
+	pid := ctx.Params("pid")
+	if pid == "" {
+		return fmt.Errorf("pid is empty")
+	}
+	if err := s.ctl.FetchArticleDetail(ctx.Context(), pid); err != nil {
+		return err
+	}
+	return ctx.JSON(web.NewResponse("ok"))
+}
+
+func (s *SpiderSvc) FetchArticleAts(ctx fiber.Ctx) error {
+	pid := ctx.Params("pid")
+	if pid == "" {
+		return fmt.Errorf("pid is empty")
+	}
+	if err := s.ctl.FetchArticleAts(ctx.Context(), pid); err != nil {
+		return err
+	}
+	return ctx.JSON(web.NewResponse("ok"))
+}
+
 //func (s *SpiderSvc) DelCalculate(ctx fiber.Ctx) error {
 //	idStr := ctx.Params("id")
 //	if idStr == "" {

spider/controller.go

@@ -145,6 +145,48 @@ func (c *Controller) AutoMigrate() error {
 	return c.storage.ProviderArticle().AutoMigrate()
 }
 
+func (c *Controller) productsToArticles(products []coach_client.Product) (articles []v2.Article) {
+	for _, product := range products {
+		// 一个color是一个sku
+		for _, color := range product.Colors {
+			// 如果没找到，说明没有标准商品，创建一个
+			article := v2.Article{
+				Name:        product.Name,
+				EnglishName: product.Name,
+				Pid:         color.VgId,
+				Brand:       v2.Brand_Coach,
+				Image:       color.Media.Thumbnail.Src,
+				Providers:   make([]v2.ProviderArticle, 0, 1),
+			}
+			pArticle := v2.ProviderArticle{
+				ProviderId: c.providerId,
+				Brand:      v2.Brand_Coach,
+				Pid:        color.VgId,
+				SkuID:      color.VgId,
+				Available:  color.Orderable,
+				Link:       fmt.Sprintf("%s/%s", "https://www.coachoutlet.com", color.Url),
+			}
+			// 拿到现在的价格
+			price, _ := lo.Find(product.VariantsOnSale, func(item coach_client.Variant) bool {
+				return item.Id == color.VgId
+			})
+			// 计算成本
+			pArticle.Cost = utils.CalculateProviderPrice(
+				append(c.provider.CalculateProcess, pArticle.CalculateProcess...),
+				map[string]float64{
+					"originalPrice": price.Price.Sales.Value,
+					"freight":       c.provider.Config.Freight,
+					"exchangeRate":  c.provider.Config.ExchangeRate,
+				})
+			pArticle.HistoryPrice = append(pArticle.HistoryPrice, pArticle.Cost)
+			article.Providers = append(article.Providers, pArticle)
+			articles = append(articles, article)
+		}
+	}
+	return
+}
+
+// TODO长时间没更新的，需要更新
 func (c *Controller) Crawl() {
 	glog.Infof("%s 开始抓取信息", time.Now())
 	// 开始拉取，修改状态
@@ -161,7 +203,7 @@ func (c *Controller) Crawl() {
 			continue
 		}
 		totalPage = resp.PageData.TotalPages
-		c.saveProducts(resp.PageData.Products)
+		c.saveProducts(c.productsToArticles(resp.PageData.Products))
 		glog.Infof("第%d页数据保存完成", page)
 		break
 	}
@@ -177,78 +219,98 @@ func (c *Controller) Crawl() {
 }
 
 // 对coach返回的数据进行处理保存
-// TODO 如果供应商的价格比商品现在的最低价格低，那么需要通知计算商品的利润率
+func (c *Controller) saveProducts(articles []v2.Article) {
-func (c *Controller) saveProducts(list []coach_client.Product) {
+	for _, article := range articles {
-	for _, resp := range list {
+		oldArticle, err := c.storage.Article().Get(storage.NewGetArticleQuery().SetBrand(v2.Brand_Coach).SetPid(article.Pid))
-		// 一个color是一个sku
+		if err != nil && !errors.As(err, &gorm.ErrRecordNotFound) {
-		for _, color := range resp.Colors {
+			glog.Errorf("获取商品信息失败: %v", err)
-			article, err := c.storage.Article().Get(storage.NewGetArticleQuery().SetBrand(v2.Brand_Coach).SetPid(color.VgId))
+			continue
-			if err != nil && !errors.As(err, &gorm.ErrRecordNotFound) {
+		}
-				glog.Errorf("获取商品信息失败: %v", err)
+		// 如果已经存在了，那么不需要重新创建article
-				continue
+		if err == nil {
-			}
+			// 查看现在有没有这个供应商的商品
-			// 如果没找到，说明没有标准商品，创建一个
+			oldProviderArticle, _, exist := lo.FindIndexOf(oldArticle.Providers, func(item v2.ProviderArticle) bool {
-			if errors.As(err, &gorm.ErrRecordNotFound) {
+				return item.ProviderId == c.providerId
-				article = v2.Article{
+			})
-					Name:        resp.Name,
+			// 创建供应商商品
-					EnglishName: resp.Name,
+			if !exist {
-					Pid:         color.VgId,
+				oldArticle.Providers = append(oldArticle.Providers, article.Providers...)
-					Brand:       v2.Brand_Coach,
+				// 保存商品信息
-					Image:       color.Media.Thumbnail.Src,
+				if err = c.storage.Article().Upsert(article); err != nil {
-					Providers:   make([]v2.ProviderArticle, 0, 1),
+					glog.Errorf("保存商品信息失败: %v", err)
+					continue
 				}
-				if err = c.storage.Article().Create(&article); err != nil {
+			} else {
-					glog.Errorf("创建商品信息失败: %v", err)
+				pArticle := article.Providers[0]
+				if oldProviderArticle.Cost.OriginalPrice != pArticle.Cost.OriginalPrice {
+					oldProviderArticle.HistoryPrice = append(oldProviderArticle.HistoryPrice, pArticle.Cost)
+				}
+				oldProviderArticle.Cost = pArticle.Cost
+				oldProviderArticle.Ast = pArticle.Ast
+				if err = c.storage.ProviderArticle().Upsert(oldProviderArticle); err != nil {
+					glog.Errorf("保存供应商商品信息失败: %v", err)
 					continue
 				}
 			}
-			pArticle, idx, exist := lo.FindIndexOf(article.Providers, func(item v2.ProviderArticle) bool {
-				return item.ProviderId == c.providerId
-			})
-			// 创建供应商上坪
-			if !exist {
-				pArticle = v2.ProviderArticle{
-					ProviderId: c.providerId,
-					Brand:      v2.Brand_Coach,
-					Pid:        color.VgId,
-					SkuID:      color.VgId,
-					Available:  color.Orderable,
-					Link:       fmt.Sprintf("%s/%s", "https://www.coachoutlet.com", color.Url),
-				}
-			} else {
-				article.Providers = lo.DropByIndex(article.Providers, idx)
-			}
-
-			// 拿到现在的价格
-			price, _ := lo.Find(resp.VariantsOnSale, func(item coach_client.Variant) bool {
-				return item.Id == color.VgId
-			})
-
-			if originalPrice := price.Price.Sales.Value; originalPrice != pArticle.Cost.OriginalPrice {
-				// 价格发生了改变
-				pArticle.Cost = utils.CalculateProviderPrice(
-					append(c.provider.CalculateProcess, pArticle.CalculateProcess...),
-					map[string]float64{
-						"originalPrice": originalPrice,
-						"freight":       c.provider.Config.Freight,
-						"exchangeRate":  c.provider.Config.ExchangeRate,
-					})
-				pArticle.HistoryPrice = append(pArticle.HistoryPrice, pArticle.Cost)
-			}
-			article.Providers = append(article.Providers, pArticle)
 
+		} else {
+			// 如果article不存在，那么保存整个article
 			// 保存商品信息
 			if err = c.storage.Article().Upsert(article); err != nil {
 				glog.Errorf("保存商品信息失败: %v", err)
 				continue
 			}
-			if err = c.subscribeClient.Publish(c.ctx, utils.ProfitRate_Channel, strconv.Itoa(int(article.ID))); err != nil {
+		}
-				glog.Errorf("通知商品利润率失败: %v", err)
+
-			}
+		if err = c.subscribeClient.Publish(c.ctx, utils.ProfitRate_Channel, strconv.Itoa(int(article.ID))); err != nil {
+			glog.Errorf("通知商品利润率失败: %v", err)
 		}
 	}
 }
 
+func (c *Controller) FetchArticleDetail(ctx context.Context, pid string) error {
+	resp, err := c.client.RequestProductDetail(ctx, pid)
+	if err != nil {
+		return fmt.Errorf("请求商品信息失败: %v", err)
+	}
+
+	article := v2.Article{
+		Name:        resp.Name,
+		EnglishName: resp.Name,
+		Pid:         resp.Id,
+		Brand:       v2.Brand_Coach,
+		Providers: []v2.ProviderArticle{
+			{
+				ProviderId: c.providerId,
+				Brand:      v2.Brand_Coach,
+				Pid:        pid,
+				SkuID:      pid,
+				Available:  resp.Inventory.Orderable,
+				Ast:        resp.Inventory.Ats,
+				Link:       fmt.Sprintf("%s/%s", "https://www.coachoutlet.com", resp.Url),
+			},
+		},
+	}
+	if len(resp.ImageGroups) > 0 {
+		article.Image = resp.ImageGroups[0].Images[0].Src
+	}
+	c.saveProducts([]v2.Article{article})
+	return nil
+}
+
+func (c *Controller)FetchArticleAts(ctx context.Context, pid string) error {
+	pArticle, err := c.storage.ProviderArticle().Get(storage.NewGetProviderArticleQuery().SetProviderId(c.providerId).SetPid(pid))
+	if err != nil {
+		return fmt.Errorf("获取商品信息失败: %v", err)
+	}
+	inv, err := c.client.RequestInventory(ctx, pid)
+	if err != nil {
+		return fmt.Errorf("请求商品库存失败: %v", err)
+	}
+	pArticle.Ast = inv.Ats
+	return c.storage.ProviderArticle().Update(pArticle, "ast")
+}
+
 // 更新某个商品的价格
 func (c *Controller) FetchArticlePrice(ctx context.Context, id uint) error {
 	pArticle, err := c.storage.ProviderArticle().Get(storage.NewGetProviderArticleQuery().SetProviderId(c.providerId).SetID(id))