kedaya_haitao/us-coach-spider
1
0
Fork 0
Code Issues 3 Pull Requests Actions Packages Projects Releases 1 Wiki Activity

feat 支持抓取美国coach
All checks were successful
Build image / build (push) Successful in 2m32s
Details

Browse Source
This commit is contained in:
timerzz 2024-12-06 17:24:57 +08:00
parent df799e679a
commit 4770d1cdf6
7 changed files with 20 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
ats-tracer/controller.go
View File

@ -24,14 +24,14 @@ type Controller struct {
tracers *Tracers
storage *storage.Storage
client *coach_client.US
client coach_client.USClient
providerId v2.ProviderId
interval time.Duration
threshold int
}
func NewController(ctx context.Context, cfg *options.Config, client *coach_client.US, db *gorm.DB) *Controller {
func NewController(ctx context.Context, cfg *options.Config, client coach_client.USClient, db *gorm.DB) *Controller {
return &Controller{
ctx: ctx,
providerId: cfg.ProviderId,

2
cmd/main.go
View File

@ -58,7 +58,7 @@ func main() {
go pool.CronUpdate(ctx) //定时更新代理
// coach client
cli := coach_client.USClient(pool)
cli := coach_client.US(pool).CreateByType(cfg.ClientType)
var wg, _ctx = errgroup.WithContext(ctx)

4
go.mod
View File

@ -5,7 +5,7 @@ go 1.22.2
toolchain go1.22.3
require (
gitea.timerzz.com/kedaya_haitao/common v0.0.0-20241204113257-ab2c85f42fe0
gitea.timerzz.com/kedaya_haitao/common v0.0.0-20241206091919-ee3f7ce41b28
gitea.timerzz.com/kedaya_haitao/pusher v0.0.0-20241129135359-c16e02a7eab0
github.com/cloudwego/kitex v0.11.3
github.com/gofiber/fiber/v3 v3.0.0-beta.3
@ -14,7 +14,6 @@ require (
github.com/redis/go-redis/v9 v9.6.1
github.com/samber/lo v1.47.0
golang.org/x/sync v0.9.0
gopkg.in/yaml.v3 v3.0.1
gorm.io/gorm v1.25.10
)
@ -180,6 +179,7 @@ require (
google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f // indirect
google.golang.org/protobuf v1.33.0 // indirect
gopkg.in/sourcemap.v1 v1.0.5 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
gorm.io/driver/postgres v1.5.7 // indirect
lukechampine.com/blake3 v1.2.2 // indirect
)

8
go.sum
View File

@ -1,10 +1,6 @@
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
gitea.timerzz.com/kedaya_haitao/common v0.0.0-20241203012724-2cc7b5cfb70a h1:BfHVMbA29/0KLg4Nv0PQVnca1pwJp7EQqjsth6A0CBY=
gitea.timerzz.com/kedaya_haitao/common v0.0.0-20241203012724-2cc7b5cfb70a/go.mod h1:BIz+IMGznPiyLnV1+Ntw1zf8rEIcbymmGq+EfvDsSgE=
gitea.timerzz.com/kedaya_haitao/common v0.0.0-20241204113022-f3d77b581683 h1:nYEQnTZ00MGP5RgfgYLj0GTJdFlMoLbcIVFH+FyYPwE=
gitea.timerzz.com/kedaya_haitao/common v0.0.0-20241204113022-f3d77b581683/go.mod h1:BIz+IMGznPiyLnV1+Ntw1zf8rEIcbymmGq+EfvDsSgE=
gitea.timerzz.com/kedaya_haitao/common v0.0.0-20241204113257-ab2c85f42fe0 h1:ND9BfeFQVXXd14nzSuJuDGBIrTYPcMvWQVkZ/C2Wc6M=
gitea.timerzz.com/kedaya_haitao/common v0.0.0-20241204113257-ab2c85f42fe0/go.mod h1:BIz+IMGznPiyLnV1+Ntw1zf8rEIcbymmGq+EfvDsSgE=
gitea.timerzz.com/kedaya_haitao/common v0.0.0-20241206091919-ee3f7ce41b28 h1:d/J5jPO0KWglNBH01Ch3ewIq2McrvKyOeTykOz6UaZo=
gitea.timerzz.com/kedaya_haitao/common v0.0.0-20241206091919-ee3f7ce41b28/go.mod h1:BIz+IMGznPiyLnV1+Ntw1zf8rEIcbymmGq+EfvDsSgE=
gitea.timerzz.com/kedaya_haitao/pusher v0.0.0-20241129135359-c16e02a7eab0 h1:WMNOErbI6At865VWI3sN74RMQaZ8ZhwsNSB9A4vg/6Q=
gitea.timerzz.com/kedaya_haitao/pusher v0.0.0-20241129135359-c16e02a7eab0/go.mod h1:nRdxwOP3hhkUdH3PjHq3gt8SA+YEfR/d7Ig9DuQQZQY=
github.com/3andne/restls-client-go v0.1.6 h1:tRx/YilqW7iHpgmEL4E1D8dAsuB0tFF3uvncS+B6I08=

5
pkg/options/init.go
View File

@ -6,6 +6,7 @@ import (
"strconv"
"time"
coach_client "gitea.timerzz.com/kedaya_haitao/common/pkg/coach-client"
v2 "gitea.timerzz.com/kedaya_haitao/common/structs/v2"
"github.com/golang/glog"
)
@ -15,6 +16,7 @@ type Config struct {
WatchInterval time.Duration
AtsInterval time.Duration
AtsThreshold int // 库存一定时间内减少多少个通知
ClientType string
}
func LoadConfigs() (opt *Config, err error) {
@ -37,7 +39,8 @@ func LoadConfigs() (opt *Config, err error) {
if opt.AtsThreshold == 0 {
opt.AtsThreshold = 40
}
opt.ClientType = coach_client.LoadUSClientType()
glog.Infof("加载watch interval %s\nats interval %s\nats threshold %d", opt.WatchInterval, opt.AtsInterval, opt.AtsThreshold)
glog.Infof("加载watch interval %s\nats interval %s\nats threshold %d\nclient type %s", opt.WatchInterval, opt.AtsInterval, opt.AtsThreshold, opt.ClientType)
return
}

11
spider/controller.go
View File

@ -3,6 +3,7 @@ package spider
import (
"context"
"fmt"
"net/url"
"strconv"
"strings"
"time"
@ -24,7 +25,7 @@ type Controller struct {
ctx context.Context
cron *cron.Cron
storage *storage.Storage
client *coach_client.US
client coach_client.USClient
rdb *redis.Client
sub *subscribe.Server
providerId v2.ProviderId
@ -33,7 +34,7 @@ type Controller struct {
ready bool
}
func NewController(ctx context.Context, providerId v2.ProviderId, client *coach_client.US, db *gorm.DB, rdb *redis.Client) *Controller {
func NewController(ctx context.Context, providerId v2.ProviderId, client coach_client.USClient, db *gorm.DB, rdb *redis.Client) *Controller {
return &Controller{
ctx: ctx,
providerId: providerId,
@ -158,6 +159,7 @@ func (c *Controller) productsToArticles(products []coach_client.Product) (articl
Image: color.Media.Thumbnail.Src,
Providers: make([]v2.ProviderArticle, 0, 1),
}
link, _ := url.JoinPath(c.client.BaseUrl(), color.Url)
pArticle := v2.ProviderArticle{
ProviderId: c.providerId,
Brand: v2.Brand_Coach,
@ -165,7 +167,7 @@ func (c *Controller) productsToArticles(products []coach_client.Product) (articl
SkuID: color.VgId,
Available: color.Orderable,
Image: color.Media.Thumbnail.Src,
Link: fmt.Sprintf("%s/%s", "https://www.coachoutlet.com", color.Url),
Link: link,
}
// 拿到现在的价格
price, _ := lo.Find(product.VariantsOnSale, func(item coach_client.Variant) bool {
@ -297,6 +299,7 @@ func (c *Controller) FetchArticleDetail(ctx context.Context, pid string) error {
if len(resp.ImageGroups) > 0 {
article.Image = resp.ImageGroups[0].Images[0].Src
}
link, _ := url.JoinPath(c.client.BaseUrl(), resp.Url)
pArticle := v2.ProviderArticle{
ProviderId: c.providerId,
Brand: v2.Brand_Coach,
@ -305,7 +308,7 @@ func (c *Controller) FetchArticleDetail(ctx context.Context, pid string) error {
Available: resp.Inventory.Orderable,
Ats: resp.Inventory.Ats,
Image: article.Image,
Link: fmt.Sprintf("%s/%s", "https://www.coachoutlet.com", resp.Url),
Link: link,
Cost: utils.CalculateProviderPrice(
append(c.provider.CalculateProcess, old.CalculateProcess...),
map[string]float64{

4
watcher/controller.go
View File

@ -24,13 +24,13 @@ type Controller struct {
watchers *Watchers
storage *storage.Storage
client *coach_client.US
client coach_client.USClient
providerId v2.ProviderId
interval time.Duration
}
func NewController(ctx context.Context, cfg *options.Config, client *coach_client.US, db *gorm.DB) *Controller {
func NewController(ctx context.Context, cfg *options.Config, client coach_client.USClient, db *gorm.DB) *Controller {
return &Controller{
ctx: ctx,
providerId: cfg.ProviderId,