kedaya_haitao/dw-spider
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat v2
Some checks failed
Build image / build (push) Failing after 25s
Details

Browse Source
This commit is contained in:
timerzz 2024-09-13 22:52:21 +08:00
parent b9c4669e83
commit ec26c06f4a
8 changed files with 505 additions and 211 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.gitea/workflows/build-push.yml
View File

@ -14,11 +14,11 @@ jobs:
with: with:
version: v0.15.4 version: v0.15.4
env: env:
KO_DOCKER_REPO: 192.168.31.55:5000/kedaya/dw-spider KO_DOCKER_REPO: 192.168.31.55:5000/kedaya/dw-spider:v2
HTTP_PROXY: http://192.168.31.55:10809 HTTP_PROXY: http://192.168.31.55:10809
HTTPS_PROXY: http://192.168.31.55:10809 HTTPS_PROXY: http://192.168.31.55:10809
- run: ko build --bare ./cmd - run: ko build --bare ./cmd
env: env:
KO_DOCKER_REPO: 192.168.31.55:5000/kedaya/dw-spider KO_DOCKER_REPO: 192.168.31.55:5000/kedaya/dw-spider:v2
HTTP_PROXY: http://192.168.31.55:10809 HTTP_PROXY: http://192.168.31.55:10809
HTTPS_PROXY: http://192.168.31.55:10809 HTTPS_PROXY: http://192.168.31.55:10809

97
cmd/main.go
View File

@ -2,55 +2,98 @@ package main
import ( import (
"context" "context"
"log" "flag"
"log/slog"
"os" "os"
"os/signal" "os/signal"
"time"
"gitea.timerzz.com/kedaya_haitao/common/pkg/database" "gitea.timerzz.com/kedaya_haitao/common/pkg/database"
"gitea.timerzz.com/kedaya_haitao/common/pkg/redis"
"gitea.timerzz.com/kedaya_haitao/common/pkg/web"
"gitea.timerzz.com/kedaya_haitao/common/structs/storage"
v2 "gitea.timerzz.com/kedaya_haitao/common/structs/v2"
dw_sdk "gitea.timerzz.com/kedaya_haitao/dw-sdk" dw_sdk "gitea.timerzz.com/kedaya_haitao/dw-sdk"
"gitea.timerzz.com/kedaya_haitao/dw-spider/options" "gitea.timerzz.com/kedaya_haitao/dw-spider/server"
"gitea.timerzz.com/kedaya_haitao/dw-spider/spider" "gitea.timerzz.com/kedaya_haitao/dw-spider/spider"
"github.com/gofiber/fiber/v3" "github.com/gofiber/fiber/v3"
"github.com/gofiber/fiber/v3/middleware/cors" "github.com/gofiber/fiber/v3/middleware/cors"
"github.com/gofiber/fiber/v3/middleware/recover"
"github.com/golang/glog" "github.com/golang/glog"
"gorm.io/gorm/logger" "golang.org/x/sync/errgroup"
) )
func main() { func main() {
slog.Info(">>> BEGIN INIT<<<") flag.Parse()
glog.Info(">>> BEGIN INIT<<<")
ctx, cancel := signal.NotifyContext(context.Background(), os.Kill, os.Interrupt) ctx, cancel := signal.NotifyContext(context.Background(), os.Kill, os.Interrupt)
defer cancel() defer cancel()
cfg, err := options.LoadConfig()
if err != nil { // 读取环境变量
glog.Fatalf("获取配置失败：%v", err) sellerId := v2.SellerId(os.Getenv("SELLER_ID"))
if sellerId == "" {
glog.Fatal("SELLER_ID 未设置")
} }
db, err := database.InitDatabase(&cfg.DB)
bidType := dw_sdk.LoadBiddingType(os.Getenv("BID_TYPE"))
db, err := database.InitDefaultDatabase()
if err != nil { if err != nil {
glog.Fatalf("初始化数据库失败：%v", err) glog.Fatalf("初始化数据库失败：%v", err)
} }
db.Logger = logger.New( // 初始化redis
log.New(os.Stdout, "\r\n", log.LstdFlags), rdb, err := redis.InitDefaultRedis()
logger.Config{ if err != nil {
SlowThreshold: time.Second, glog.Fatalf("初始化redis失败：%v", err)
LogLevel: logger.Warn, }
IgnoreRecordNotFoundError: false, client, err := dw_sdk.InitDefaultDWClient()
Colorful: true, if err != nil {
}, glog.Fatalf("初始化redis失败：%v", err)
}
// wait group
var wg, _ctx = errgroup.WithContext(ctx)
// 控制器
ctl := spider.NewController(
ctx,
sellerId,
client,
storage.NewStorage(db),
rdb,
bidType,
) )
client := dw_sdk.NewClient(cfg.Dw) wg.Go(ctl.Run)
ctl := spider.NewController(client, db)
go ctl.Run(ctx)
r := fiber.New()
r.Use(cors.New())
r.Get("/health", func(ctx fiber.Ctx) error { // server
return ctx.SendString("ok") r := fiber.New(fiber.Config{ErrorHandler: web.ErrHandle})
r.Use(cors.New(), recover.New())
for _, register := range []web.Register{
server.NewSpiderSvc(ctl, sellerId),
web.NewProbe().SetReadyProbe(func(ctx fiber.Ctx) error {
if ctl.Ready() {
return ctx.SendStatus(fiber.StatusOK)
} else {
return ctx.SendStatus(fiber.StatusServiceUnavailable)
}
}),
} {
register.Registry(r)
}
glog.Info(">>> BEGIN RUN<<<")
port := os.Getenv("PORT")
if port == "" {
port = "80"
}
wg.Go(func() error {
defer glog.Infof("server 服务退出")
return r.Listen(":"+port, fiber.ListenConfig{
EnablePrintRoutes: true,
GracefulContext: _ctx,
})
}) })
if err = r.Listen(":8080"); err != nil { if err = wg.Wait(); err != nil {
glog.Warningf("server over: %v", err) glog.Fatalf("服务异常退出：%v", err)
} }
} }

34
go.mod
View File

@ -3,23 +3,28 @@ module gitea.timerzz.com/kedaya_haitao/dw-spider
go 1.22.5 go 1.22.5
require ( require (
gitea.timerzz.com/kedaya_haitao/common v0.0.0-20240729073435-c498a70be1e1 gitea.timerzz.com/kedaya_haitao/common v0.0.0-20240913121100-f0a601466ba1
gitea.timerzz.com/kedaya_haitao/dw-sdk v0.0.0-20240729074513-b04e8891dc86 gitea.timerzz.com/kedaya_haitao/dw-sdk v0.0.0-20240904075121-552ef11ea87c
github.com/gofiber/fiber/v3 v3.0.0-beta.3 github.com/gofiber/fiber/v3 v3.0.0-beta.3
github.com/golang/glog v1.2.1 github.com/golang/glog v1.2.1
gopkg.in/yaml.v3 v3.0.1 github.com/pkg/errors v0.9.1
gorm.io/gorm v1.25.10 github.com/redis/go-redis/v9 v9.6.1
github.com/samber/lo v1.39.0
golang.org/x/sync v0.8.0
gorm.io/gorm v1.25.11
) )
require ( require (
github.com/andybalholm/brotli v1.1.0 // indirect github.com/andybalholm/brotli v1.1.0 // indirect
github.com/bytedance/sonic v1.11.9 // indirect github.com/bytedance/sonic v1.12.2 // indirect
github.com/bytedance/sonic/loader v0.1.1 // indirect github.com/bytedance/sonic/loader v0.2.0 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/cloudwego/base64x v0.1.4 // indirect github.com/cloudwego/base64x v0.1.4 // indirect
github.com/cloudwego/iasm v0.2.0 // indirect github.com/cloudwego/iasm v0.2.0 // indirect
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
github.com/expr-lang/expr v1.16.9 // indirect github.com/expr-lang/expr v1.16.9 // indirect
github.com/gofiber/utils/v2 v2.0.0-beta.4 // indirect github.com/gofiber/utils/v2 v2.0.0-beta.6 // indirect
github.com/google/go-querystring v1.1.0 // indirect github.com/google/go-querystring v1.1.0 // indirect
github.com/google/uuid v1.6.0 // indirect github.com/google/uuid v1.6.0 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect
@ -28,19 +33,20 @@ require (
github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/inflection v1.0.0 // indirect
github.com/jinzhu/now v1.1.5 // indirect github.com/jinzhu/now v1.1.5 // indirect
github.com/klauspost/compress v1.17.9 // indirect github.com/klauspost/compress v1.17.9 // indirect
github.com/klauspost/cpuid/v2 v2.2.7 // indirect github.com/klauspost/cpuid/v2 v2.2.8 // indirect
github.com/kr/pretty v0.3.1 // indirect github.com/kr/pretty v0.3.1 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-isatty v0.0.20 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect
github.com/valyala/fasthttp v1.55.0 // indirect github.com/valyala/fasthttp v1.55.0 // indirect
github.com/valyala/tcplisten v1.0.0 // indirect github.com/valyala/tcplisten v1.0.0 // indirect
golang.org/x/arch v0.3.0 // indirect golang.org/x/arch v0.9.0 // indirect
golang.org/x/crypto v0.24.0 // indirect golang.org/x/crypto v0.26.0 // indirect
golang.org/x/net v0.26.0 // indirect golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f // indirect
golang.org/x/sys v0.21.0 // indirect golang.org/x/net v0.28.0 // indirect
golang.org/x/text v0.16.0 // indirect golang.org/x/sys v0.24.0 // indirect
golang.org/x/text v0.17.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
gorm.io/driver/postgres v1.5.7 // indirect gorm.io/driver/postgres v1.5.7 // indirect
) )

65
go.sum
View File

@ -1,13 +1,20 @@
gitea.timerzz.com/kedaya_haitao/common v0.0.0-20240729073435-c498a70be1e1 h1:TBm8q1PriPd1OuHYiKAz3/kS0q4Ay1MotT49Cq9hPPM= gitea.timerzz.com/kedaya_haitao/common v0.0.0-20240913121100-f0a601466ba1 h1:5Bu3V2w4Jd7mgFIeH8GkVaEfvOXIFtPSoApwzsmp9rc=
gitea.timerzz.com/kedaya_haitao/common v0.0.0-20240729073435-c498a70be1e1/go.mod h1:uV586p6Z8LIq3I8O/pXZv+jIkIwnwBjAz0D7KrhB9JM= gitea.timerzz.com/kedaya_haitao/common v0.0.0-20240913121100-f0a601466ba1/go.mod h1:BIz+IMGznPiyLnV1+Ntw1zf8rEIcbymmGq+EfvDsSgE=
gitea.timerzz.com/kedaya_haitao/dw-sdk v0.0.0-20240729074513-b04e8891dc86 h1:W2/xPenk+ST7wmtKl6UO4qiLstT3wjMY9OY79bN8J8Q= gitea.timerzz.com/kedaya_haitao/dw-sdk v0.0.0-20240904075121-552ef11ea87c h1:pami6fFsRqKYJyJbSGqXaz2hsbgims7/3ma7cY9CwuY=
gitea.timerzz.com/kedaya_haitao/dw-sdk v0.0.0-20240729074513-b04e8891dc86/go.mod h1:d4UIdrlnAsRoYo55TB8J46FlnpY00psHSO85TMWw3d4= gitea.timerzz.com/kedaya_haitao/dw-sdk v0.0.0-20240904075121-552ef11ea87c/go.mod h1:Yfl45xbT2dTh4YHqz4E2i++TYq7fahTeSv9JWNFLjNo=
github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M= github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M=
github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY= github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY=
github.com/bytedance/sonic v1.11.9 h1:LFHENlIY/SLzDWverzdOvgMztTxcfcF+cqNsz9pK5zg= github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
github.com/bytedance/sonic v1.11.9/go.mod h1:LysEHSvpvDySVdC2f87zGWf6CIKJcAvqab1ZaiQtds4= github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
github.com/bytedance/sonic/loader v0.1.1 h1:c+e5Pt1k/cy5wMveRDyk2X4B9hF4g7an8N3zCYjJFNM= github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
github.com/bytedance/sonic v1.12.2 h1:oaMFuRTpMHYLpCntGca65YWt5ny+wAceDERTkT2L9lg=
github.com/bytedance/sonic v1.12.2/go.mod h1:B8Gt/XvtZ3Fqj+iSKMypzymZxw/FVwgIGKzMzT9r/rk=
github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU=
github.com/bytedance/sonic/loader v0.2.0 h1:zNprn+lsIP06C/IqCHs3gPQIvnvpKbbxyXQP1iU4kWM=
github.com/bytedance/sonic/loader v0.2.0/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU=
github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/0Y= github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/0Y=
github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w=
github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg= github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg=
@ -17,12 +24,14 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
github.com/expr-lang/expr v1.16.9 h1:WUAzmR0JNI9JCiF0/ewwHB1gmcGw5wW7nWt8gc6PpCI= github.com/expr-lang/expr v1.16.9 h1:WUAzmR0JNI9JCiF0/ewwHB1gmcGw5wW7nWt8gc6PpCI=
github.com/expr-lang/expr v1.16.9/go.mod h1:8/vRC7+7HBzESEqt5kKpYXxrxkr31SaO8r40VO/1IT4= github.com/expr-lang/expr v1.16.9/go.mod h1:8/vRC7+7HBzESEqt5kKpYXxrxkr31SaO8r40VO/1IT4=
github.com/gofiber/fiber/v3 v3.0.0-beta.3 h1:7Q2I+HsIqnIEEDB+9oe7Gadpakh6ZLhXpTYz/L20vrg= github.com/gofiber/fiber/v3 v3.0.0-beta.3 h1:7Q2I+HsIqnIEEDB+9oe7Gadpakh6ZLhXpTYz/L20vrg=
github.com/gofiber/fiber/v3 v3.0.0-beta.3/go.mod h1:kcMur0Dxqk91R7p4vxEpJfDWZ9u5IfvrtQc8Bvv/JmY= github.com/gofiber/fiber/v3 v3.0.0-beta.3/go.mod h1:kcMur0Dxqk91R7p4vxEpJfDWZ9u5IfvrtQc8Bvv/JmY=
github.com/gofiber/utils/v2 v2.0.0-beta.4 h1:1gjbVFFwVwUb9arPcqiB6iEjHBwo7cHsyS41NeIW3co= github.com/gofiber/utils/v2 v2.0.0-beta.6 h1:ED62bOmpRXdgviPlfTmf0Q+AXzhaTUAFtdWjgx+XkYI=
github.com/gofiber/utils/v2 v2.0.0-beta.4/go.mod h1:sdRsPU1FXX6YiDGGxd+q2aPJRMzpsxdzCXo9dz+xtOY= github.com/gofiber/utils/v2 v2.0.0-beta.6/go.mod h1:3Kz8Px3jInKFvqxDzDeoSygwEOO+3uyubTmUa6PqY+0=
github.com/golang/glog v1.2.1 h1:OptwRhECazUx5ix5TTWC3EZhsZEHWcYWY4FQHTIubm4= github.com/golang/glog v1.2.1 h1:OptwRhECazUx5ix5TTWC3EZhsZEHWcYWY4FQHTIubm4=
github.com/golang/glog v1.2.1/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= github.com/golang/glog v1.2.1/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w=
github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
@ -45,8 +54,8 @@ github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/
github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=
github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM= github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM=
github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
@ -62,9 +71,13 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/redis/go-redis/v9 v9.6.1 h1:HHDteefn6ZkTtY5fGUE8tj8uy85AHk6zP7CpzIAM0y4=
github.com/redis/go-redis/v9 v9.6.1/go.mod h1:0C0c6ycQsdpVNQpxb1njEQIqkx5UcsM8FJCQLgE9+RA=
github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
github.com/samber/lo v1.39.0 h1:4gTz1wUhNYLhFSKl6O+8peW0v2F4BCY034GRpU9WnuA=
github.com/samber/lo v1.39.0/go.mod h1:+m/ZKRl6ClXCE2Lgf3MsQlWfh4bn1bz6CXEOxnEXnEA=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
@ -83,20 +96,23 @@ github.com/valyala/fasthttp v1.55.0 h1:Zkefzgt6a7+bVKHnu/YaYSOPfNYNisSVBo/unVCf8
github.com/valyala/fasthttp v1.55.0/go.mod h1:NkY9JtkrpPKmgwV3HTaS2HWaJss9RSIsRVfcxxoHiOM= github.com/valyala/fasthttp v1.55.0/go.mod h1:NkY9JtkrpPKmgwV3HTaS2HWaJss9RSIsRVfcxxoHiOM=
github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8= github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8=
github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc= github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc=
golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/arch v0.9.0 h1:ub9TgUInamJ8mrZIGlBG6/4TqWeMszd4N8lNorbrr6k=
golang.org/x/arch v0.3.0 h1:02VY4/ZcO/gBOH6PUaoiptASxtXU10jazRCP865E97k= golang.org/x/arch v0.9.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys=
golang.org/x/arch v0.3.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw=
golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54=
golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f h1:99ci1mjWVBWwJiEKYY6jWa4d2nTQVIEhZIptnrVb1XY=
golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f/go.mod h1:/lliqkxwWAhPjf5oSOIJup2XcqJaw8RGS6k3TGEc7GI=
golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE=
golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg=
golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ=
golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg=
golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc=
golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
@ -106,7 +122,6 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gorm.io/driver/postgres v1.5.7 h1:8ptbNJTDbEmhdr62uReG5BGkdQyeasu/FZHxI0IMGnM= gorm.io/driver/postgres v1.5.7 h1:8ptbNJTDbEmhdr62uReG5BGkdQyeasu/FZHxI0IMGnM=
gorm.io/driver/postgres v1.5.7/go.mod h1:3e019WlBaYI5o5LIdNV+LyxCMNtLOQETBXL2h4chKpA= gorm.io/driver/postgres v1.5.7/go.mod h1:3e019WlBaYI5o5LIdNV+LyxCMNtLOQETBXL2h4chKpA=
gorm.io/gorm v1.25.10 h1:dQpO+33KalOA+aFYGlK+EfxcI5MbO7EP2yYygwh9h+s= gorm.io/gorm v1.25.11 h1:/Wfyg1B/je1hnDx3sMkX+gAlxrlZpn6X0BXRlwXlvHg=
gorm.io/gorm v1.25.10/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= gorm.io/gorm v1.25.11/go.mod h1:xh7N7RHfYlNc5EmcI/El95gXusucDrQnHXe0+CgWcLQ=
nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50=
rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=

28
options/cfg.go
View File

@ -1,28 +0,0 @@
package options
import (
"os"
"gitea.timerzz.com/kedaya_haitao/common/pkg/database"
dw_sdk "gitea.timerzz.com/kedaya_haitao/dw-sdk"
"gopkg.in/yaml.v3"
)
type Config struct {
DB database.DBOption `yaml:"db"`
Dw dw_sdk.Config `yaml:"dw"`
}
func LoadConfig() (*Config, error) {
cfgPath := os.Getenv("CONFIG_PATH")
if cfgPath == "" {
cfgPath = "/data/cfg.yaml"
}
var opt Config
f, err := os.Open(cfgPath)
if err != nil {
return nil, err
}
defer f.Close()
return &opt, yaml.NewDecoder(f).Decode(&opt)
}

89
server/spider.go Normal file
View File

@ -0,0 +1,89 @@
package server
import (
"fmt"
"strconv"
"gitea.timerzz.com/kedaya_haitao/common/pkg/web"
v2 "gitea.timerzz.com/kedaya_haitao/common/structs/v2"
"gitea.timerzz.com/kedaya_haitao/dw-spider/spider"
"github.com/gofiber/fiber/v3"
)
type SpiderSvc struct {
ctl *spider.Controller
sellerId v2.SellerId
}
func NewSpiderSvc(ctl *spider.Controller, sellerId v2.SellerId) *SpiderSvc {
return &SpiderSvc{
ctl: ctl,
sellerId: sellerId,
}
}
func (s *SpiderSvc) Registry(r fiber.Router) {
api := r.Group("/api/v2")
api.Post(fmt.Sprintf("seller/%s/pull", s.sellerId), s.Pull)
api.Post(fmt.Sprintf("seller/%s/match", s.sellerId), s.Match)
api.Post(fmt.Sprintf("seller/%s/fetch/:id", s.sellerId), s.FetchArticlePrice)
//r.Post("spider/us/coach-outlet/calculate", s.UpsertCalculate)
//r.Delete("spider/us/coach-outlet/calculate/u/:id", s.DelCalculate)
}
func (s *SpiderSvc) Pull(ctx fiber.Ctx) error {
// 接收到，就爬取数据
go s.ctl.Crawl()
return ctx.JSON(web.NewResponse("ok", "ok"))
}
func (s *SpiderSvc) Match(ctx fiber.Ctx) error {
var article v2.SellerArticle
if err := ctx.Bind().JSON(&article); err != nil {
return err
}
if article.ID == 0 {
return fiber.ErrBadRequest
}
article, err := s.ctl.MatchDWSluId(article)
if err != nil {
return err
}
return ctx.JSON(web.NewResponse(article))
}
func (s *SpiderSvc) FetchArticlePrice(ctx fiber.Ctx) error {
i := ctx.Params("id")
id, _ := strconv.Atoi(i)
if id == 0 {
return fiber.ErrBadRequest
}
err := s.ctl.FetchArticlePrice(uint(id))
if err != nil {
return err
}
return ctx.JSON(web.NewResponse("ok"))
}
//func (s *SpiderSvc) DelCalculate(ctx fiber.Ctx) error {
// idStr := ctx.Params("id")
// if idStr == "" {
// return fiber.ErrBadRequest
// }
// id, _ := strconv.Atoi(idStr)
// if err := s.ctl.DeleteCalculateProcess(uint(id)); err != nil {
// return err
// }
// return nil
//}
//
//func (s *SpiderSvc) UpsertCalculate(ctx fiber.Ctx) error {
// var cs []productv1.CalculateProcess
// if err := ctx.Bind().JSON(&cs); err != nil {
// return err
// }
// if err := s.ctl.SaveCalculateProcess(cs); err != nil {
// return err
// }
// return nil
//}

348
spider/controller.go
View File

@ -3,152 +3,306 @@ package spider
import ( import (
"context" "context"
"fmt" "fmt"
"log/slog" "strconv"
"strings"
"time" "time"
productv1 "gitea.timerzz.com/kedaya_haitao/common/model/product" "gitea.timerzz.com/kedaya_haitao/common/pkg/cron"
"gitea.timerzz.com/kedaya_haitao/common/pkg/subscribe"
"gitea.timerzz.com/kedaya_haitao/common/structs/storage"
"gitea.timerzz.com/kedaya_haitao/common/structs/utils"
v2 "gitea.timerzz.com/kedaya_haitao/common/structs/v2"
dw_sdk "gitea.timerzz.com/kedaya_haitao/dw-sdk" dw_sdk "gitea.timerzz.com/kedaya_haitao/dw-sdk"
"github.com/golang/glog"
"github.com/pkg/errors"
"github.com/redis/go-redis/v9"
"github.com/samber/lo"
"gorm.io/gorm" "gorm.io/gorm"
) )
type Controller struct { type Controller struct {
ctx context.Context ctx context.Context
cron *cron.Cron
storage *storage.Storage
client *dw_sdk.Client client *dw_sdk.Client
updateTime time.Time // 上次抓取时间 rdb *redis.Client
db *gorm.DB // 订阅服务
sub *subscribe.Server
sellerId v2.SellerId
seller v2.Seller
bidType dw_sdk.BiddingType
bidClient dw_sdk.BidClient
// 订阅客户端
subscribeClient *subscribe.Client
ready bool
} }
func NewController(client *dw_sdk.Client, db *gorm.DB) *Controller { func NewController(ctx context.Context, sellerId v2.SellerId, client *dw_sdk.Client, storage *storage.Storage, rdb *redis.Client, bidType dw_sdk.BiddingType) *Controller {
c := &Controller{ return &Controller{
ctx: ctx,
sellerId: sellerId,
client: client, client: client,
db: db, storage: storage,
} cron: cron.NewCron(),
c.AutoMigrate() rdb: rdb,
return c bidType: bidType,
} sub: subscribe.NewServer(ctx, rdb),
bidClient: client.BidClientByBidType(bidType),
func (c *Controller) AutoMigrate() { subscribeClient: subscribe.NewClient(rdb),
if err := c.db.AutoMigrate(&productv1.Product{}); err != nil {
panic(err)
} }
} }
func (c *Controller) Run(ctx context.Context) { func (c *Controller) Run() (err error) {
c.ctx = ctx if err = c.AutoMigrate(); err != nil {
ticker := time.NewTicker(24 * time.Hour) return err
if err := c.Crawl(); err != nil {
slog.Error(err.Error())
} else {
slog.Info("抓取信息成功")
c.updateTime = time.Now()
} }
for { if c.seller, err = c.storage.Seller().GetBySellerId(c.sellerId); err != nil {
select { return fmt.Errorf("没有找到对应的销售商信息：%v", err)
case <-ctx.Done():
return
case <-ticker.C:
if err := c.Crawl(); err != nil {
slog.Error(err.Error())
} else {
slog.Info("抓取信息成功")
c.updateTime = time.Now()
} }
if err = c.ListenSeller(c.ctx, c.rdb); err != nil {
return errors.Wrap(err, "订阅失败")
}
// 定时抓取
go c.cron.SetTimeHHmm(c.seller.Config.Ticker).SetFunc(c.Crawl).Run(c.ctx)
c.ready = true
<-c.ctx.Done()
glog.Infof("controller服务退出")
return nil
} }
} // 监听配置变更
} func (c *Controller) ListenSeller(ctx context.Context, rdb *redis.Client) error {
server := subscribe.NewServer(ctx, rdb).SetErrorHandle(func(err error) {
func (c *Controller) Crawl() error { glog.Errorf("销售商信息更新处理失败: %v", err)
slog.Info("开始抓取信息") })
var products []productv1.Product err := server.Subscribe(utils.SellerConfigNotifyChannel(c.sellerId), c.sellerChange)
c.db.Where("dw_sku_id!=(-1) or dw_sku_id is null").FindInBatches(&products, 50, func(tx *gorm.DB, batch int) error {
for _, product := range products {
if product.DWSkuId == 0 {
p, err := c.fillDWSkuId(product)
if err != nil { if err != nil {
slog.Warn(err.Error()) return errors.Wrap(err, "订阅失败")
} }
if err = c.db.Select("dw_sku_id", "dw_spu_id").Updates(&p).Error; err != nil { go server.Run()
slog.Warn(fmt.Sprintf("更新pid: %s sku失败：%v", p.Pid, err)) return nil
} }
product = p
if product.DWSkuId <= 0 { // 销售商信息发生变化
continue func (c *Controller) sellerChange(ctx context.Context, message string) error {
} seller, err := c.storage.Seller().GetBySellerId(c.sellerId)
}
resp, err := c.client.ConsignBidClient().LowestPrice(product.DWSkuId)
if err != nil { if err != nil {
slog.Warn(fmt.Sprintf("请求sku:%d最低价失败:%v", product.DWSkuId, err)) return err
}
// 定的拉取时间变了
if seller.Config.Ticker != c.seller.Config.Ticker {
// 重新启动下定时任务
c.cron.Stop()
go c.cron.SetTimeHHmm(seller.Config.Ticker).Run(c.ctx)
}
// 检查要不要重新计算所有sellerArticle的价格
var needUpdate = false
oldProcess := make(map[uint]v2.CalculateProcess, len(c.seller.CalculateProcess))
for _, process := range c.seller.CalculateProcess {
oldProcess[process.ID] = process
}
for _, process := range seller.CalculateProcess {
old, ok := oldProcess[process.ID]
if !ok {
// 不存在，说明新增了
needUpdate = true
break
}
if old.Condition != process.Condition || old.Process != process.Process {
// 条件或者计算过程变了，需要更新
needUpdate = true
}
delete(oldProcess, process.ID)
}
if len(oldProcess) > 0 {
needUpdate = true
}
c.seller.Config = seller.Config
c.seller.CalculateProcess = seller.CalculateProcess
if needUpdate {
var results = make([]v2.SellerArticle, 0, 20)
err = c.storage.SellerArticle().FindInBatches(storage.NewGetSellerArticleQuery().SetSellerId(c.sellerId), &results, func(tx *gorm.DB, batch int) error {
for idx := range results {
results[idx].Sell = utils.CalculateSellerPrice(append(seller.CalculateProcess, results[idx].CalculateProcess...), map[string]float64{
"originalPrice": results[idx].Sell.OriginalPrice,
})
}
return tx.Select("id", "sell").Save(&results).Error
})
}
return err
}
func (c *Controller) AutoMigrate() error {
if err := c.storage.Article().AutoMigrate(); err != nil {
return err
}
if err := c.storage.Seller().AutoMigrate(); err != nil {
return err
}
return c.storage.SellerArticle().AutoMigrate()
}
func (c *Controller) Crawl() {
glog.Infof("%s 开始抓取信息", time.Now())
// 开始拉取，修改状态
c.setSellerStatus(v2.SellerStatus_Pulling)
var msgs = make([]string, 0)
var results = make([]*v2.Article, 0, 20)
c.storage.DB().Debug().
Preload("Sellers", "seller_id = ?", c.sellerId).Preload("Sellers.CalculateProcess").
Joins("LEFT JOIN seller_articles ON seller_articles.article_id=articles.id AND seller_articles.seller_id = ?", c.sellerId).
Where("(seller_articles.exclude is null OR not seller_articles.exclude) AND (articles.exclude is null OR not articles.exclude)").
FindInBatches(&results, 20, func(tx *gorm.DB, batch int) error {
glog.Infof("正在抓取第%d批数据", batch)
for _, article := range results {
var sArticle v2.SellerArticle
if len(article.Sellers) > 0 {
sArticle = article.Sellers[0]
} else {
var err error
sArticle, err = c.MatchDWSluId(v2.SellerArticle{
ArticleID: article.ID,
SellerId: c.sellerId,
Pid: article.Pid,
Brand: article.Brand,
})
if err != nil {
glog.Warning(err)
}
if sArticle.SkuID == "" {
continue continue
} }
if resp.Code != 200 { sArticle, err = c.storage.SellerArticle().Get(storage.NewGetSellerArticleQuery().SetSellerId(c.sellerId).SetSkuId(sArticle.SkuID))
slog.Warn(fmt.Sprintf("请求sku: %d，得物返回错误：%s", product.DWSkuId, resp.Msg)) if err != nil {
glog.Warning(err)
continue continue
} }
if len(resp.Data.Items) == 0 { }
slog.Warn(fmt.Sprintf("未获取到sku: %d 的最低价,数量为0", product.DWSkuId))
if err := c.PullArticle(sArticle); err != nil {
msgs = append(msgs, err.Error())
glog.Error(err.Error())
continue continue
} }
product.DWPrice = float64(resp.Data.Items[0].LowestPrice)/100*0.925 - 46 - 8.5 // 抓取成功，利润率重新计算
product.CalRate() if err := c.subscribeClient.Publish(c.ctx, utils.ProfitRate_Channel, strconv.Itoa(int(article.ID))); err != nil {
if err = c.db.Select("rate", "dw_price").Updates(&product).Error; err != nil { glog.Warningf("发布消息失败: %v", err)
slog.Warn(fmt.Sprintf("更新pid:%s的数据库失败：%v", product.Pid, err))
} }
//if product.Rate >= 15 && product.Orderable {
// c.Push(fmt.Sprintf("%s 的收益率达到%.2f%%", product.Pid, product.Rate),
// fmt.Sprintf("%s 的成本为%.2f, 得物收益为%.2f, 收益率为%.2f%%， 链接：%s", product.Pid, product.CNYPrice, product.DWPrice, product.Rate, product.Link))
//}
continue
} }
slog.Info("已处理50个数据") glog.Infof("完成抓取第%d批数据", batch)
return nil return nil
}) })
slog.Info("抓取完毕")
return nil // 拉取结束，修改状态
c.seller.PullAt = time.Now()
if len(msgs) > 0 {
c.setSellerStatus(v2.SellerStatus_Error, strings.Join(msgs, "\n"))
} else {
c.setSellerStatus(v2.SellerStatus_Normal)
}
glog.Infof("%s 抓取信息结束", time.Now())
} }
// 填充得物 // 尝试重新匹配得物的skuid
func (c *Controller) fillDWSkuId(product productv1.Product) (productv1.Product, error) { func (c *Controller) MatchDWSluId(article v2.SellerArticle) (v2.SellerArticle, error) {
resp, err := c.client.ArticleService().BatchArticleNumber([]string{product.Pid}) resp, err := c.client.ArticleService().BatchArticleNumber([]string{article.Pid})
if err != nil { if err != nil {
return product, fmt.Errorf("请求pid:%s出错：%v", product.Pid, err) return article, fmt.Errorf("请求pid:%s出错：%v", article.Pid, err)
} }
if resp.Code != 200 { if resp.Code != 200 {
product.DWSkuId = -1 article.Exclude = true
return product, fmt.Errorf("pid:%s 得物返回错误：%s", product.Pid, resp.Msg) _ = c.storage.SellerArticle().Upsert(article)
return article, fmt.Errorf("pid:%s 得物返回错误：%s", article.Pid, resp.Msg)
} }
if len(resp.Data) == 0 { if len(resp.Data) == 0 {
product.DWSkuId = -1 article.Exclude = true
return product, fmt.Errorf("pid:%s 得物返回spu为0", product.Pid) _ = c.storage.SellerArticle().Upsert(article)
return article, fmt.Errorf("pid:%s 得物返回spu为0", article.Pid)
} }
var spu *dw_sdk.Spu var spu *dw_sdk.Spu
brandId, brandName := BrandIdMap[article.Brand], BrandNameMap[article.Brand]
for _, s := range resp.Data { for _, s := range resp.Data {
if s.BrandId == dw_sdk.BrandId_Coach { if s.BrandId == brandId || s.BrandName == brandName {
spu = &s spu = &s
} }
} }
if spu == nil { if spu == nil {
product.DWSkuId = -1 article.Exclude = true
return product, fmt.Errorf("未获取到pid : %s 的coach spu信息:%v", product.Pid, resp.Data) _ = c.storage.SellerArticle().Upsert(article)
return article, fmt.Errorf("未获取到pid : %s 的coach spu信息:%v", article.Pid, resp.Data)
} }
if len(spu.Skus) == 0 { if len(spu.Skus) == 0 {
product.DWSkuId = -1 article.Exclude = true
return product, fmt.Errorf("未获取到pid : %s 的coach sku信息:%v", product.Pid, spu) _ = c.storage.SellerArticle().Upsert(article)
return article, fmt.Errorf("未获取到pid : %s 的coach sku信息:%v", article.Pid, spu)
} }
product.DWSpuId, product.DWSkuId = spu.SpuId, spu.Skus[0].SkuId article.SpuID, article.SkuID = strconv.Itoa(spu.SpuId), strconv.Itoa(spu.Skus[0].SkuId)
return product, nil article.Exclude = false
return article, c.storage.SellerArticle().Upsert(article)
} }
//func (c *Controller) Push(title, content string) { func (c *Controller) PullArticle(sArticle v2.SellerArticle) error {
// resp, err := pusher.Push(c.ctx, &push.PushReq{ skuId, _ := strconv.Atoi(sArticle.SkuID)
// Ids: []int64{2}, resp, err := c.bidClient.LowestPrice(skuId)
// Title: title, if err != nil {
// Content: content, return fmt.Errorf("请求sku:%s最低价失败:%v", sArticle.SkuID, err)
// }) }
// if err != nil {
// glog.Errorf("消息推送失败：%v", err) lowest, exist := lo.Find(resp.Items, func(item dw_sdk.LowestPriceItem) bool {
// } return item.BiddingType == c.bidType
// if resp.Code != 0 { })
// glog.Errorf("消息推送失败:%s", resp.Msg) if !exist {
// } // 没有拿到价格，将exclude设置为true，如果后面要拉取，需要手动打开
//} sArticle.Exclude = true
return c.storage.SellerArticle().Upsert(sArticle)
}
sell := utils.CalculateSellerPrice(append(c.seller.CalculateProcess, sArticle.CalculateProcess...), map[string]float64{
"originalPrice": float64(lowest.LowestPrice / 100),
})
if sell.OriginalPrice != sArticle.Sell.OriginalPrice {
sArticle.Sell = sell
sArticle.HistoryPrice = append(sArticle.HistoryPrice, sArticle.Sell)
}
if err = c.storage.SellerArticle().Upsert(sArticle); err != nil {
return fmt.Errorf("保存sellerArticle失败:%v", err)
}
return nil
}
// 修改保存状态
func (c *Controller) setSellerStatus(status v2.SellerStatus, msg ...string) {
c.seller.Status = status
if len(msg) > 0 {
c.seller.Msg = msg[0]
}
if err := c.storage.Seller().UpdateStatus(c.seller); err != nil {
glog.Errorf("更新状态失败: %v", err)
}
}
func (c *Controller) Ready() bool {
return c.ready
}
func (c *Controller) FetchArticlePrice(id uint) error {
article, err := c.storage.SellerArticle().Get(storage.NewGetSellerArticleQuery().SetID(id))
if err != nil {
return err
}
err = c.PullArticle(article)
if err != nil {
return err
}
// 抓取成功，利润率重新计算
if err = c.subscribeClient.Publish(c.ctx, utils.ProfitRate_Channel, strconv.Itoa(int(article.ArticleID))); err != nil {
glog.Warningf("发布消息失败: %v", err)
}
return nil
}

15
spider/utiles.go Normal file
View File

@ -0,0 +1,15 @@
package spider
import (
v2 "gitea.timerzz.com/kedaya_haitao/common/structs/v2"
dw_sdk "gitea.timerzz.com/kedaya_haitao/dw-sdk"
)
var (
BrandIdMap = map[v2.Brand]dw_sdk.BrandId{
v2.Brand_Coach: dw_sdk.BrandId_Coach,
}
BrandNameMap = map[v2.Brand]dw_sdk.BrandName{
v2.Brand_Coach: dw_sdk.BrandName_Coach,
}
)