This commit is contained in:
parent
b9c4669e83
commit
ec26c06f4a
@ -14,11 +14,11 @@ jobs:
|
||||
with:
|
||||
version: v0.15.4
|
||||
env:
|
||||
KO_DOCKER_REPO: 192.168.31.55:5000/kedaya/dw-spider
|
||||
KO_DOCKER_REPO: 192.168.31.55:5000/kedaya/dw-spider:v2
|
||||
HTTP_PROXY: http://192.168.31.55:10809
|
||||
HTTPS_PROXY: http://192.168.31.55:10809
|
||||
- run: ko build --bare ./cmd
|
||||
env:
|
||||
KO_DOCKER_REPO: 192.168.31.55:5000/kedaya/dw-spider
|
||||
KO_DOCKER_REPO: 192.168.31.55:5000/kedaya/dw-spider:v2
|
||||
HTTP_PROXY: http://192.168.31.55:10809
|
||||
HTTPS_PROXY: http://192.168.31.55:10809
|
||||
97
cmd/main.go
97
cmd/main.go
@ -2,55 +2,98 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"log/slog"
|
||||
"flag"
|
||||
"os"
|
||||
"os/signal"
|
||||
"time"
|
||||
|
||||
"gitea.timerzz.com/kedaya_haitao/common/pkg/database"
|
||||
"gitea.timerzz.com/kedaya_haitao/common/pkg/redis"
|
||||
"gitea.timerzz.com/kedaya_haitao/common/pkg/web"
|
||||
"gitea.timerzz.com/kedaya_haitao/common/structs/storage"
|
||||
v2 "gitea.timerzz.com/kedaya_haitao/common/structs/v2"
|
||||
dw_sdk "gitea.timerzz.com/kedaya_haitao/dw-sdk"
|
||||
"gitea.timerzz.com/kedaya_haitao/dw-spider/options"
|
||||
"gitea.timerzz.com/kedaya_haitao/dw-spider/server"
|
||||
"gitea.timerzz.com/kedaya_haitao/dw-spider/spider"
|
||||
"github.com/gofiber/fiber/v3"
|
||||
"github.com/gofiber/fiber/v3/middleware/cors"
|
||||
"github.com/gofiber/fiber/v3/middleware/recover"
|
||||
"github.com/golang/glog"
|
||||
"gorm.io/gorm/logger"
|
||||
"golang.org/x/sync/errgroup"
|
||||
)
|
||||
|
||||
func main() {
|
||||
slog.Info(">>> BEGIN INIT<<<")
|
||||
flag.Parse()
|
||||
glog.Info(">>> BEGIN INIT<<<")
|
||||
ctx, cancel := signal.NotifyContext(context.Background(), os.Kill, os.Interrupt)
|
||||
defer cancel()
|
||||
cfg, err := options.LoadConfig()
|
||||
if err != nil {
|
||||
glog.Fatalf("获取配置失败：%v", err)
|
||||
|
||||
// 读取环境变量
|
||||
sellerId := v2.SellerId(os.Getenv("SELLER_ID"))
|
||||
if sellerId == "" {
|
||||
glog.Fatal("SELLER_ID 未设置")
|
||||
}
|
||||
db, err := database.InitDatabase(&cfg.DB)
|
||||
|
||||
bidType := dw_sdk.LoadBiddingType(os.Getenv("BID_TYPE"))
|
||||
|
||||
db, err := database.InitDefaultDatabase()
|
||||
if err != nil {
|
||||
glog.Fatalf("初始化数据库失败：%v", err)
|
||||
}
|
||||
db.Logger = logger.New(
|
||||
log.New(os.Stdout, "\r\n", log.LstdFlags),
|
||||
logger.Config{
|
||||
SlowThreshold: time.Second,
|
||||
LogLevel: logger.Warn,
|
||||
IgnoreRecordNotFoundError: false,
|
||||
Colorful: true,
|
||||
},
|
||||
// 初始化redis
|
||||
rdb, err := redis.InitDefaultRedis()
|
||||
if err != nil {
|
||||
glog.Fatalf("初始化redis失败：%v", err)
|
||||
}
|
||||
client, err := dw_sdk.InitDefaultDWClient()
|
||||
if err != nil {
|
||||
glog.Fatalf("初始化redis失败：%v", err)
|
||||
}
|
||||
// wait group
|
||||
var wg, _ctx = errgroup.WithContext(ctx)
|
||||
// 控制器
|
||||
ctl := spider.NewController(
|
||||
ctx,
|
||||
sellerId,
|
||||
client,
|
||||
storage.NewStorage(db),
|
||||
rdb,
|
||||
bidType,
|
||||
)
|
||||
|
||||
client := dw_sdk.NewClient(cfg.Dw)
|
||||
ctl := spider.NewController(client, db)
|
||||
go ctl.Run(ctx)
|
||||
r := fiber.New()
|
||||
r.Use(cors.New())
|
||||
wg.Go(ctl.Run)
|
||||
|
||||
r.Get("/health", func(ctx fiber.Ctx) error {
|
||||
return ctx.SendString("ok")
|
||||
// server
|
||||
r := fiber.New(fiber.Config{ErrorHandler: web.ErrHandle})
|
||||
|
||||
r.Use(cors.New(), recover.New())
|
||||
|
||||
for _, register := range []web.Register{
|
||||
server.NewSpiderSvc(ctl, sellerId),
|
||||
web.NewProbe().SetReadyProbe(func(ctx fiber.Ctx) error {
|
||||
if ctl.Ready() {
|
||||
return ctx.SendStatus(fiber.StatusOK)
|
||||
} else {
|
||||
return ctx.SendStatus(fiber.StatusServiceUnavailable)
|
||||
}
|
||||
}),
|
||||
} {
|
||||
register.Registry(r)
|
||||
}
|
||||
glog.Info(">>> BEGIN RUN<<<")
|
||||
port := os.Getenv("PORT")
|
||||
if port == "" {
|
||||
port = "80"
|
||||
}
|
||||
|
||||
wg.Go(func() error {
|
||||
defer glog.Infof("server 服务退出")
|
||||
return r.Listen(":"+port, fiber.ListenConfig{
|
||||
EnablePrintRoutes: true,
|
||||
GracefulContext: _ctx,
|
||||
})
|
||||
})
|
||||
|
||||
if err = r.Listen(":8080"); err != nil {
|
||||
glog.Warningf("server over: %v", err)
|
||||
if err = wg.Wait(); err != nil {
|
||||
glog.Fatalf("服务异常退出：%v", err)
|
||||
}
|
||||
}
|
||||
|
||||
34
go.mod
34
go.mod
@ -3,23 +3,28 @@ module gitea.timerzz.com/kedaya_haitao/dw-spider
|
||||
go 1.22.5
|
||||
|
||||
require (
|
||||
gitea.timerzz.com/kedaya_haitao/common v0.0.0-20240729073435-c498a70be1e1
|
||||
gitea.timerzz.com/kedaya_haitao/dw-sdk v0.0.0-20240729074513-b04e8891dc86
|
||||
gitea.timerzz.com/kedaya_haitao/common v0.0.0-20240913121100-f0a601466ba1
|
||||
gitea.timerzz.com/kedaya_haitao/dw-sdk v0.0.0-20240904075121-552ef11ea87c
|
||||
github.com/gofiber/fiber/v3 v3.0.0-beta.3
|
||||
github.com/golang/glog v1.2.1
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
gorm.io/gorm v1.25.10
|
||||
github.com/pkg/errors v0.9.1
|
||||
github.com/redis/go-redis/v9 v9.6.1
|
||||
github.com/samber/lo v1.39.0
|
||||
golang.org/x/sync v0.8.0
|
||||
gorm.io/gorm v1.25.11
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/andybalholm/brotli v1.1.0 // indirect
|
||||
github.com/bytedance/sonic v1.11.9 // indirect
|
||||
github.com/bytedance/sonic/loader v0.1.1 // indirect
|
||||
github.com/bytedance/sonic v1.12.2 // indirect
|
||||
github.com/bytedance/sonic/loader v0.2.0 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.2.0 // indirect
|
||||
github.com/cloudwego/base64x v0.1.4 // indirect
|
||||
github.com/cloudwego/iasm v0.2.0 // indirect
|
||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
|
||||
github.com/expr-lang/expr v1.16.9 // indirect
|
||||
github.com/gofiber/utils/v2 v2.0.0-beta.4 // indirect
|
||||
github.com/gofiber/utils/v2 v2.0.0-beta.6 // indirect
|
||||
github.com/google/go-querystring v1.1.0 // indirect
|
||||
github.com/google/uuid v1.6.0 // indirect
|
||||
github.com/jackc/pgpassfile v1.0.0 // indirect
|
||||
@ -28,19 +33,20 @@ require (
|
||||
github.com/jinzhu/inflection v1.0.0 // indirect
|
||||
github.com/jinzhu/now v1.1.5 // indirect
|
||||
github.com/klauspost/compress v1.17.9 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.2.7 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.2.8 // indirect
|
||||
github.com/kr/pretty v0.3.1 // indirect
|
||||
github.com/mattn/go-colorable v0.1.13 // indirect
|
||||
github.com/mattn/go-isatty v0.0.20 // indirect
|
||||
github.com/pkg/errors v0.9.1 // indirect
|
||||
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
|
||||
github.com/valyala/bytebufferpool v1.0.0 // indirect
|
||||
github.com/valyala/fasthttp v1.55.0 // indirect
|
||||
github.com/valyala/tcplisten v1.0.0 // indirect
|
||||
golang.org/x/arch v0.3.0 // indirect
|
||||
golang.org/x/crypto v0.24.0 // indirect
|
||||
golang.org/x/net v0.26.0 // indirect
|
||||
golang.org/x/sys v0.21.0 // indirect
|
||||
golang.org/x/text v0.16.0 // indirect
|
||||
golang.org/x/arch v0.9.0 // indirect
|
||||
golang.org/x/crypto v0.26.0 // indirect
|
||||
golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f // indirect
|
||||
golang.org/x/net v0.28.0 // indirect
|
||||
golang.org/x/sys v0.24.0 // indirect
|
||||
golang.org/x/text v0.17.0 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
gorm.io/driver/postgres v1.5.7 // indirect
|
||||
)
|
||||
|
||||
65
go.sum
65
go.sum
@ -1,13 +1,20 @@
|
||||
gitea.timerzz.com/kedaya_haitao/common v0.0.0-20240729073435-c498a70be1e1 h1:TBm8q1PriPd1OuHYiKAz3/kS0q4Ay1MotT49Cq9hPPM=
|
||||
gitea.timerzz.com/kedaya_haitao/common v0.0.0-20240729073435-c498a70be1e1/go.mod h1:uV586p6Z8LIq3I8O/pXZv+jIkIwnwBjAz0D7KrhB9JM=
|
||||
gitea.timerzz.com/kedaya_haitao/dw-sdk v0.0.0-20240729074513-b04e8891dc86 h1:W2/xPenk+ST7wmtKl6UO4qiLstT3wjMY9OY79bN8J8Q=
|
||||
gitea.timerzz.com/kedaya_haitao/dw-sdk v0.0.0-20240729074513-b04e8891dc86/go.mod h1:d4UIdrlnAsRoYo55TB8J46FlnpY00psHSO85TMWw3d4=
|
||||
gitea.timerzz.com/kedaya_haitao/common v0.0.0-20240913121100-f0a601466ba1 h1:5Bu3V2w4Jd7mgFIeH8GkVaEfvOXIFtPSoApwzsmp9rc=
|
||||
gitea.timerzz.com/kedaya_haitao/common v0.0.0-20240913121100-f0a601466ba1/go.mod h1:BIz+IMGznPiyLnV1+Ntw1zf8rEIcbymmGq+EfvDsSgE=
|
||||
gitea.timerzz.com/kedaya_haitao/dw-sdk v0.0.0-20240904075121-552ef11ea87c h1:pami6fFsRqKYJyJbSGqXaz2hsbgims7/3ma7cY9CwuY=
|
||||
gitea.timerzz.com/kedaya_haitao/dw-sdk v0.0.0-20240904075121-552ef11ea87c/go.mod h1:Yfl45xbT2dTh4YHqz4E2i++TYq7fahTeSv9JWNFLjNo=
|
||||
github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M=
|
||||
github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY=
|
||||
github.com/bytedance/sonic v1.11.9 h1:LFHENlIY/SLzDWverzdOvgMztTxcfcF+cqNsz9pK5zg=
|
||||
github.com/bytedance/sonic v1.11.9/go.mod h1:LysEHSvpvDySVdC2f87zGWf6CIKJcAvqab1ZaiQtds4=
|
||||
github.com/bytedance/sonic/loader v0.1.1 h1:c+e5Pt1k/cy5wMveRDyk2X4B9hF4g7an8N3zCYjJFNM=
|
||||
github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
|
||||
github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
|
||||
github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
|
||||
github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
|
||||
github.com/bytedance/sonic v1.12.2 h1:oaMFuRTpMHYLpCntGca65YWt5ny+wAceDERTkT2L9lg=
|
||||
github.com/bytedance/sonic v1.12.2/go.mod h1:B8Gt/XvtZ3Fqj+iSKMypzymZxw/FVwgIGKzMzT9r/rk=
|
||||
github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU=
|
||||
github.com/bytedance/sonic/loader v0.2.0 h1:zNprn+lsIP06C/IqCHs3gPQIvnvpKbbxyXQP1iU4kWM=
|
||||
github.com/bytedance/sonic/loader v0.2.0/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU=
|
||||
github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
|
||||
github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/0Y=
|
||||
github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w=
|
||||
github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg=
|
||||
@ -17,12 +24,14 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
|
||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
|
||||
github.com/expr-lang/expr v1.16.9 h1:WUAzmR0JNI9JCiF0/ewwHB1gmcGw5wW7nWt8gc6PpCI=
|
||||
github.com/expr-lang/expr v1.16.9/go.mod h1:8/vRC7+7HBzESEqt5kKpYXxrxkr31SaO8r40VO/1IT4=
|
||||
github.com/gofiber/fiber/v3 v3.0.0-beta.3 h1:7Q2I+HsIqnIEEDB+9oe7Gadpakh6ZLhXpTYz/L20vrg=
|
||||
github.com/gofiber/fiber/v3 v3.0.0-beta.3/go.mod h1:kcMur0Dxqk91R7p4vxEpJfDWZ9u5IfvrtQc8Bvv/JmY=
|
||||
github.com/gofiber/utils/v2 v2.0.0-beta.4 h1:1gjbVFFwVwUb9arPcqiB6iEjHBwo7cHsyS41NeIW3co=
|
||||
github.com/gofiber/utils/v2 v2.0.0-beta.4/go.mod h1:sdRsPU1FXX6YiDGGxd+q2aPJRMzpsxdzCXo9dz+xtOY=
|
||||
github.com/gofiber/utils/v2 v2.0.0-beta.6 h1:ED62bOmpRXdgviPlfTmf0Q+AXzhaTUAFtdWjgx+XkYI=
|
||||
github.com/gofiber/utils/v2 v2.0.0-beta.6/go.mod h1:3Kz8Px3jInKFvqxDzDeoSygwEOO+3uyubTmUa6PqY+0=
|
||||
github.com/golang/glog v1.2.1 h1:OptwRhECazUx5ix5TTWC3EZhsZEHWcYWY4FQHTIubm4=
|
||||
github.com/golang/glog v1.2.1/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w=
|
||||
github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
@ -45,8 +54,8 @@ github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/
|
||||
github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=
|
||||
github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
|
||||
github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
||||
github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM=
|
||||
github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
|
||||
github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM=
|
||||
github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
|
||||
github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M=
|
||||
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
||||
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
||||
@ -62,9 +71,13 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
|
||||
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/redis/go-redis/v9 v9.6.1 h1:HHDteefn6ZkTtY5fGUE8tj8uy85AHk6zP7CpzIAM0y4=
|
||||
github.com/redis/go-redis/v9 v9.6.1/go.mod h1:0C0c6ycQsdpVNQpxb1njEQIqkx5UcsM8FJCQLgE9+RA=
|
||||
github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
|
||||
github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
|
||||
github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
|
||||
github.com/samber/lo v1.39.0 h1:4gTz1wUhNYLhFSKl6O+8peW0v2F4BCY034GRpU9WnuA=
|
||||
github.com/samber/lo v1.39.0/go.mod h1:+m/ZKRl6ClXCE2Lgf3MsQlWfh4bn1bz6CXEOxnEXnEA=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
|
||||
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
|
||||
@ -83,20 +96,23 @@ github.com/valyala/fasthttp v1.55.0 h1:Zkefzgt6a7+bVKHnu/YaYSOPfNYNisSVBo/unVCf8
|
||||
github.com/valyala/fasthttp v1.55.0/go.mod h1:NkY9JtkrpPKmgwV3HTaS2HWaJss9RSIsRVfcxxoHiOM=
|
||||
github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8=
|
||||
github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc=
|
||||
golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
|
||||
golang.org/x/arch v0.3.0 h1:02VY4/ZcO/gBOH6PUaoiptASxtXU10jazRCP865E97k=
|
||||
golang.org/x/arch v0.3.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
|
||||
golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI=
|
||||
golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM=
|
||||
golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ=
|
||||
golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE=
|
||||
golang.org/x/arch v0.9.0 h1:ub9TgUInamJ8mrZIGlBG6/4TqWeMszd4N8lNorbrr6k=
|
||||
golang.org/x/arch v0.9.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys=
|
||||
golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw=
|
||||
golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54=
|
||||
golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f h1:99ci1mjWVBWwJiEKYY6jWa4d2nTQVIEhZIptnrVb1XY=
|
||||
golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f/go.mod h1:/lliqkxwWAhPjf5oSOIJup2XcqJaw8RGS6k3TGEc7GI=
|
||||
golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE=
|
||||
golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg=
|
||||
golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ=
|
||||
golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws=
|
||||
golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
|
||||
golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
|
||||
golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg=
|
||||
golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc=
|
||||
golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
|
||||
@ -106,7 +122,6 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gorm.io/driver/postgres v1.5.7 h1:8ptbNJTDbEmhdr62uReG5BGkdQyeasu/FZHxI0IMGnM=
|
||||
gorm.io/driver/postgres v1.5.7/go.mod h1:3e019WlBaYI5o5LIdNV+LyxCMNtLOQETBXL2h4chKpA=
|
||||
gorm.io/gorm v1.25.10 h1:dQpO+33KalOA+aFYGlK+EfxcI5MbO7EP2yYygwh9h+s=
|
||||
gorm.io/gorm v1.25.10/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8=
|
||||
gorm.io/gorm v1.25.11 h1:/Wfyg1B/je1hnDx3sMkX+gAlxrlZpn6X0BXRlwXlvHg=
|
||||
gorm.io/gorm v1.25.11/go.mod h1:xh7N7RHfYlNc5EmcI/El95gXusucDrQnHXe0+CgWcLQ=
|
||||
nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50=
|
||||
rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=
|
||||
|
||||
@ -1,28 +0,0 @@
|
||||
package options
|
||||
|
||||
import (
|
||||
"os"
|
||||
|
||||
"gitea.timerzz.com/kedaya_haitao/common/pkg/database"
|
||||
dw_sdk "gitea.timerzz.com/kedaya_haitao/dw-sdk"
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
DB database.DBOption `yaml:"db"`
|
||||
Dw dw_sdk.Config `yaml:"dw"`
|
||||
}
|
||||
|
||||
func LoadConfig() (*Config, error) {
|
||||
cfgPath := os.Getenv("CONFIG_PATH")
|
||||
if cfgPath == "" {
|
||||
cfgPath = "/data/cfg.yaml"
|
||||
}
|
||||
var opt Config
|
||||
f, err := os.Open(cfgPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer f.Close()
|
||||
return &opt, yaml.NewDecoder(f).Decode(&opt)
|
||||
}
|
||||
89
server/spider.go
Normal file
89
server/spider.go
Normal file
@ -0,0 +1,89 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strconv"
|
||||
|
||||
"gitea.timerzz.com/kedaya_haitao/common/pkg/web"
|
||||
v2 "gitea.timerzz.com/kedaya_haitao/common/structs/v2"
|
||||
"gitea.timerzz.com/kedaya_haitao/dw-spider/spider"
|
||||
"github.com/gofiber/fiber/v3"
|
||||
)
|
||||
|
||||
type SpiderSvc struct {
|
||||
ctl *spider.Controller
|
||||
sellerId v2.SellerId
|
||||
}
|
||||
|
||||
func NewSpiderSvc(ctl *spider.Controller, sellerId v2.SellerId) *SpiderSvc {
|
||||
return &SpiderSvc{
|
||||
ctl: ctl,
|
||||
sellerId: sellerId,
|
||||
}
|
||||
}
|
||||
|
||||
func (s *SpiderSvc) Registry(r fiber.Router) {
|
||||
api := r.Group("/api/v2")
|
||||
api.Post(fmt.Sprintf("seller/%s/pull", s.sellerId), s.Pull)
|
||||
api.Post(fmt.Sprintf("seller/%s/match", s.sellerId), s.Match)
|
||||
api.Post(fmt.Sprintf("seller/%s/fetch/:id", s.sellerId), s.FetchArticlePrice)
|
||||
//r.Post("spider/us/coach-outlet/calculate", s.UpsertCalculate)
|
||||
//r.Delete("spider/us/coach-outlet/calculate/u/:id", s.DelCalculate)
|
||||
}
|
||||
|
||||
func (s *SpiderSvc) Pull(ctx fiber.Ctx) error {
|
||||
// 接收到，就爬取数据
|
||||
go s.ctl.Crawl()
|
||||
return ctx.JSON(web.NewResponse("ok", "ok"))
|
||||
}
|
||||
|
||||
func (s *SpiderSvc) Match(ctx fiber.Ctx) error {
|
||||
var article v2.SellerArticle
|
||||
if err := ctx.Bind().JSON(&article); err != nil {
|
||||
return err
|
||||
}
|
||||
if article.ID == 0 {
|
||||
return fiber.ErrBadRequest
|
||||
}
|
||||
article, err := s.ctl.MatchDWSluId(article)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return ctx.JSON(web.NewResponse(article))
|
||||
}
|
||||
|
||||
func (s *SpiderSvc) FetchArticlePrice(ctx fiber.Ctx) error {
|
||||
i := ctx.Params("id")
|
||||
id, _ := strconv.Atoi(i)
|
||||
if id == 0 {
|
||||
return fiber.ErrBadRequest
|
||||
}
|
||||
err := s.ctl.FetchArticlePrice(uint(id))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return ctx.JSON(web.NewResponse("ok"))
|
||||
}
|
||||
|
||||
//func (s *SpiderSvc) DelCalculate(ctx fiber.Ctx) error {
|
||||
// idStr := ctx.Params("id")
|
||||
// if idStr == "" {
|
||||
// return fiber.ErrBadRequest
|
||||
// }
|
||||
// id, _ := strconv.Atoi(idStr)
|
||||
// if err := s.ctl.DeleteCalculateProcess(uint(id)); err != nil {
|
||||
// return err
|
||||
// }
|
||||
// return nil
|
||||
//}
|
||||
//
|
||||
//func (s *SpiderSvc) UpsertCalculate(ctx fiber.Ctx) error {
|
||||
// var cs []productv1.CalculateProcess
|
||||
// if err := ctx.Bind().JSON(&cs); err != nil {
|
||||
// return err
|
||||
// }
|
||||
// if err := s.ctl.SaveCalculateProcess(cs); err != nil {
|
||||
// return err
|
||||
// }
|
||||
// return nil
|
||||
//}
|
||||
@ -3,152 +3,306 @@ package spider
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
productv1 "gitea.timerzz.com/kedaya_haitao/common/model/product"
|
||||
"gitea.timerzz.com/kedaya_haitao/common/pkg/cron"
|
||||
"gitea.timerzz.com/kedaya_haitao/common/pkg/subscribe"
|
||||
"gitea.timerzz.com/kedaya_haitao/common/structs/storage"
|
||||
"gitea.timerzz.com/kedaya_haitao/common/structs/utils"
|
||||
v2 "gitea.timerzz.com/kedaya_haitao/common/structs/v2"
|
||||
dw_sdk "gitea.timerzz.com/kedaya_haitao/dw-sdk"
|
||||
"github.com/golang/glog"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/redis/go-redis/v9"
|
||||
"github.com/samber/lo"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
type Controller struct {
|
||||
ctx context.Context
|
||||
client *dw_sdk.Client
|
||||
updateTime time.Time // 上次抓取时间
|
||||
db *gorm.DB
|
||||
ctx context.Context
|
||||
cron *cron.Cron
|
||||
storage *storage.Storage
|
||||
client *dw_sdk.Client
|
||||
rdb *redis.Client
|
||||
// 订阅服务
|
||||
sub *subscribe.Server
|
||||
sellerId v2.SellerId
|
||||
seller v2.Seller
|
||||
bidType dw_sdk.BiddingType
|
||||
bidClient dw_sdk.BidClient
|
||||
// 订阅客户端
|
||||
subscribeClient *subscribe.Client
|
||||
|
||||
ready bool
|
||||
}
|
||||
|
||||
func NewController(client *dw_sdk.Client, db *gorm.DB) *Controller {
|
||||
c := &Controller{
|
||||
client: client,
|
||||
db: db,
|
||||
}
|
||||
c.AutoMigrate()
|
||||
return c
|
||||
}
|
||||
|
||||
func (c *Controller) AutoMigrate() {
|
||||
if err := c.db.AutoMigrate(&productv1.Product{}); err != nil {
|
||||
panic(err)
|
||||
func NewController(ctx context.Context, sellerId v2.SellerId, client *dw_sdk.Client, storage *storage.Storage, rdb *redis.Client, bidType dw_sdk.BiddingType) *Controller {
|
||||
return &Controller{
|
||||
ctx: ctx,
|
||||
sellerId: sellerId,
|
||||
client: client,
|
||||
storage: storage,
|
||||
cron: cron.NewCron(),
|
||||
rdb: rdb,
|
||||
bidType: bidType,
|
||||
sub: subscribe.NewServer(ctx, rdb),
|
||||
bidClient: client.BidClientByBidType(bidType),
|
||||
subscribeClient: subscribe.NewClient(rdb),
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Controller) Run(ctx context.Context) {
|
||||
c.ctx = ctx
|
||||
ticker := time.NewTicker(24 * time.Hour)
|
||||
if err := c.Crawl(); err != nil {
|
||||
slog.Error(err.Error())
|
||||
} else {
|
||||
slog.Info("抓取信息成功")
|
||||
c.updateTime = time.Now()
|
||||
func (c *Controller) Run() (err error) {
|
||||
if err = c.AutoMigrate(); err != nil {
|
||||
return err
|
||||
}
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
if err := c.Crawl(); err != nil {
|
||||
slog.Error(err.Error())
|
||||
} else {
|
||||
slog.Info("抓取信息成功")
|
||||
c.updateTime = time.Now()
|
||||
}
|
||||
}
|
||||
|
||||
if c.seller, err = c.storage.Seller().GetBySellerId(c.sellerId); err != nil {
|
||||
return fmt.Errorf("没有找到对应的销售商信息：%v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Controller) Crawl() error {
|
||||
slog.Info("开始抓取信息")
|
||||
var products []productv1.Product
|
||||
c.db.Where("dw_sku_id!=(-1) or dw_sku_id is null").FindInBatches(&products, 50, func(tx *gorm.DB, batch int) error {
|
||||
for _, product := range products {
|
||||
if product.DWSkuId == 0 {
|
||||
p, err := c.fillDWSkuId(product)
|
||||
if err != nil {
|
||||
slog.Warn(err.Error())
|
||||
}
|
||||
if err = c.db.Select("dw_sku_id", "dw_spu_id").Updates(&p).Error; err != nil {
|
||||
slog.Warn(fmt.Sprintf("更新pid: %s sku失败：%v", p.Pid, err))
|
||||
}
|
||||
product = p
|
||||
if product.DWSkuId <= 0 {
|
||||
continue
|
||||
}
|
||||
}
|
||||
resp, err := c.client.ConsignBidClient().LowestPrice(product.DWSkuId)
|
||||
if err != nil {
|
||||
slog.Warn(fmt.Sprintf("请求sku:%d最低价失败:%v", product.DWSkuId, err))
|
||||
continue
|
||||
}
|
||||
if resp.Code != 200 {
|
||||
slog.Warn(fmt.Sprintf("请求sku: %d，得物返回错误：%s", product.DWSkuId, resp.Msg))
|
||||
continue
|
||||
}
|
||||
if len(resp.Data.Items) == 0 {
|
||||
slog.Warn(fmt.Sprintf("未获取到sku: %d 的最低价,数量为0", product.DWSkuId))
|
||||
continue
|
||||
}
|
||||
product.DWPrice = float64(resp.Data.Items[0].LowestPrice)/100*0.925 - 46 - 8.5
|
||||
product.CalRate()
|
||||
if err = c.db.Select("rate", "dw_price").Updates(&product).Error; err != nil {
|
||||
slog.Warn(fmt.Sprintf("更新pid:%s的数据库失败：%v", product.Pid, err))
|
||||
}
|
||||
//if product.Rate >= 15 && product.Orderable {
|
||||
// c.Push(fmt.Sprintf("%s 的收益率达到%.2f%%", product.Pid, product.Rate),
|
||||
// fmt.Sprintf("%s 的成本为%.2f, 得物收益为%.2f, 收益率为%.2f%%， 链接：%s", product.Pid, product.CNYPrice, product.DWPrice, product.Rate, product.Link))
|
||||
//}
|
||||
continue
|
||||
}
|
||||
slog.Info("已处理50个数据")
|
||||
return nil
|
||||
})
|
||||
slog.Info("抓取完毕")
|
||||
if err = c.ListenSeller(c.ctx, c.rdb); err != nil {
|
||||
return errors.Wrap(err, "订阅失败")
|
||||
}
|
||||
// 定时抓取
|
||||
go c.cron.SetTimeHHmm(c.seller.Config.Ticker).SetFunc(c.Crawl).Run(c.ctx)
|
||||
c.ready = true
|
||||
<-c.ctx.Done()
|
||||
glog.Infof("controller服务退出")
|
||||
return nil
|
||||
}
|
||||
|
||||
// 填充得物
|
||||
func (c *Controller) fillDWSkuId(product productv1.Product) (productv1.Product, error) {
|
||||
resp, err := c.client.ArticleService().BatchArticleNumber([]string{product.Pid})
|
||||
// 监听配置变更
|
||||
func (c *Controller) ListenSeller(ctx context.Context, rdb *redis.Client) error {
|
||||
server := subscribe.NewServer(ctx, rdb).SetErrorHandle(func(err error) {
|
||||
glog.Errorf("销售商信息更新处理失败: %v", err)
|
||||
})
|
||||
err := server.Subscribe(utils.SellerConfigNotifyChannel(c.sellerId), c.sellerChange)
|
||||
if err != nil {
|
||||
return product, fmt.Errorf("请求pid:%s出错：%v", product.Pid, err)
|
||||
return errors.Wrap(err, "订阅失败")
|
||||
}
|
||||
go server.Run()
|
||||
return nil
|
||||
}
|
||||
|
||||
// 销售商信息发生变化
|
||||
func (c *Controller) sellerChange(ctx context.Context, message string) error {
|
||||
seller, err := c.storage.Seller().GetBySellerId(c.sellerId)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// 定的拉取时间变了
|
||||
if seller.Config.Ticker != c.seller.Config.Ticker {
|
||||
// 重新启动下定时任务
|
||||
c.cron.Stop()
|
||||
go c.cron.SetTimeHHmm(seller.Config.Ticker).Run(c.ctx)
|
||||
}
|
||||
|
||||
// 检查要不要重新计算所有sellerArticle的价格
|
||||
var needUpdate = false
|
||||
|
||||
oldProcess := make(map[uint]v2.CalculateProcess, len(c.seller.CalculateProcess))
|
||||
for _, process := range c.seller.CalculateProcess {
|
||||
oldProcess[process.ID] = process
|
||||
}
|
||||
for _, process := range seller.CalculateProcess {
|
||||
old, ok := oldProcess[process.ID]
|
||||
if !ok {
|
||||
// 不存在，说明新增了
|
||||
needUpdate = true
|
||||
break
|
||||
}
|
||||
if old.Condition != process.Condition || old.Process != process.Process {
|
||||
// 条件或者计算过程变了，需要更新
|
||||
needUpdate = true
|
||||
}
|
||||
delete(oldProcess, process.ID)
|
||||
}
|
||||
if len(oldProcess) > 0 {
|
||||
needUpdate = true
|
||||
}
|
||||
c.seller.Config = seller.Config
|
||||
c.seller.CalculateProcess = seller.CalculateProcess
|
||||
|
||||
if needUpdate {
|
||||
var results = make([]v2.SellerArticle, 0, 20)
|
||||
err = c.storage.SellerArticle().FindInBatches(storage.NewGetSellerArticleQuery().SetSellerId(c.sellerId), &results, func(tx *gorm.DB, batch int) error {
|
||||
for idx := range results {
|
||||
results[idx].Sell = utils.CalculateSellerPrice(append(seller.CalculateProcess, results[idx].CalculateProcess...), map[string]float64{
|
||||
"originalPrice": results[idx].Sell.OriginalPrice,
|
||||
})
|
||||
}
|
||||
return tx.Select("id", "sell").Save(&results).Error
|
||||
})
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (c *Controller) AutoMigrate() error {
|
||||
if err := c.storage.Article().AutoMigrate(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := c.storage.Seller().AutoMigrate(); err != nil {
|
||||
return err
|
||||
}
|
||||
return c.storage.SellerArticle().AutoMigrate()
|
||||
}
|
||||
|
||||
func (c *Controller) Crawl() {
|
||||
glog.Infof("%s 开始抓取信息", time.Now())
|
||||
// 开始拉取，修改状态
|
||||
c.setSellerStatus(v2.SellerStatus_Pulling)
|
||||
|
||||
var msgs = make([]string, 0)
|
||||
var results = make([]*v2.Article, 0, 20)
|
||||
c.storage.DB().Debug().
|
||||
Preload("Sellers", "seller_id = ?", c.sellerId).Preload("Sellers.CalculateProcess").
|
||||
Joins("LEFT JOIN seller_articles ON seller_articles.article_id=articles.id AND seller_articles.seller_id = ?", c.sellerId).
|
||||
Where("(seller_articles.exclude is null OR not seller_articles.exclude) AND (articles.exclude is null OR not articles.exclude)").
|
||||
FindInBatches(&results, 20, func(tx *gorm.DB, batch int) error {
|
||||
glog.Infof("正在抓取第%d批数据", batch)
|
||||
for _, article := range results {
|
||||
var sArticle v2.SellerArticle
|
||||
if len(article.Sellers) > 0 {
|
||||
sArticle = article.Sellers[0]
|
||||
} else {
|
||||
var err error
|
||||
sArticle, err = c.MatchDWSluId(v2.SellerArticle{
|
||||
ArticleID: article.ID,
|
||||
SellerId: c.sellerId,
|
||||
Pid: article.Pid,
|
||||
Brand: article.Brand,
|
||||
})
|
||||
if err != nil {
|
||||
glog.Warning(err)
|
||||
}
|
||||
|
||||
if sArticle.SkuID == "" {
|
||||
continue
|
||||
}
|
||||
sArticle, err = c.storage.SellerArticle().Get(storage.NewGetSellerArticleQuery().SetSellerId(c.sellerId).SetSkuId(sArticle.SkuID))
|
||||
if err != nil {
|
||||
glog.Warning(err)
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
if err := c.PullArticle(sArticle); err != nil {
|
||||
msgs = append(msgs, err.Error())
|
||||
glog.Error(err.Error())
|
||||
continue
|
||||
}
|
||||
// 抓取成功，利润率重新计算
|
||||
if err := c.subscribeClient.Publish(c.ctx, utils.ProfitRate_Channel, strconv.Itoa(int(article.ID))); err != nil {
|
||||
glog.Warningf("发布消息失败: %v", err)
|
||||
}
|
||||
}
|
||||
glog.Infof("完成抓取第%d批数据", batch)
|
||||
return nil
|
||||
})
|
||||
|
||||
// 拉取结束，修改状态
|
||||
c.seller.PullAt = time.Now()
|
||||
if len(msgs) > 0 {
|
||||
c.setSellerStatus(v2.SellerStatus_Error, strings.Join(msgs, "\n"))
|
||||
} else {
|
||||
c.setSellerStatus(v2.SellerStatus_Normal)
|
||||
}
|
||||
glog.Infof("%s 抓取信息结束", time.Now())
|
||||
}
|
||||
|
||||
// 尝试重新匹配得物的skuid
|
||||
func (c *Controller) MatchDWSluId(article v2.SellerArticle) (v2.SellerArticle, error) {
|
||||
resp, err := c.client.ArticleService().BatchArticleNumber([]string{article.Pid})
|
||||
if err != nil {
|
||||
return article, fmt.Errorf("请求pid:%s出错：%v", article.Pid, err)
|
||||
}
|
||||
if resp.Code != 200 {
|
||||
product.DWSkuId = -1
|
||||
return product, fmt.Errorf("pid:%s 得物返回错误：%s", product.Pid, resp.Msg)
|
||||
article.Exclude = true
|
||||
_ = c.storage.SellerArticle().Upsert(article)
|
||||
return article, fmt.Errorf("pid:%s 得物返回错误：%s", article.Pid, resp.Msg)
|
||||
}
|
||||
if len(resp.Data) == 0 {
|
||||
product.DWSkuId = -1
|
||||
return product, fmt.Errorf("pid:%s 得物返回spu为0", product.Pid)
|
||||
article.Exclude = true
|
||||
_ = c.storage.SellerArticle().Upsert(article)
|
||||
return article, fmt.Errorf("pid:%s 得物返回spu为0", article.Pid)
|
||||
}
|
||||
var spu *dw_sdk.Spu
|
||||
brandId, brandName := BrandIdMap[article.Brand], BrandNameMap[article.Brand]
|
||||
for _, s := range resp.Data {
|
||||
if s.BrandId == dw_sdk.BrandId_Coach {
|
||||
if s.BrandId == brandId || s.BrandName == brandName {
|
||||
spu = &s
|
||||
}
|
||||
}
|
||||
if spu == nil {
|
||||
product.DWSkuId = -1
|
||||
return product, fmt.Errorf("未获取到pid : %s 的coach spu信息:%v", product.Pid, resp.Data)
|
||||
article.Exclude = true
|
||||
_ = c.storage.SellerArticle().Upsert(article)
|
||||
return article, fmt.Errorf("未获取到pid : %s 的coach spu信息:%v", article.Pid, resp.Data)
|
||||
}
|
||||
if len(spu.Skus) == 0 {
|
||||
product.DWSkuId = -1
|
||||
return product, fmt.Errorf("未获取到pid : %s 的coach sku信息:%v", product.Pid, spu)
|
||||
article.Exclude = true
|
||||
_ = c.storage.SellerArticle().Upsert(article)
|
||||
return article, fmt.Errorf("未获取到pid : %s 的coach sku信息:%v", article.Pid, spu)
|
||||
}
|
||||
product.DWSpuId, product.DWSkuId = spu.SpuId, spu.Skus[0].SkuId
|
||||
return product, nil
|
||||
article.SpuID, article.SkuID = strconv.Itoa(spu.SpuId), strconv.Itoa(spu.Skus[0].SkuId)
|
||||
article.Exclude = false
|
||||
return article, c.storage.SellerArticle().Upsert(article)
|
||||
}
|
||||
|
||||
//func (c *Controller) Push(title, content string) {
|
||||
// resp, err := pusher.Push(c.ctx, &push.PushReq{
|
||||
// Ids: []int64{2},
|
||||
// Title: title,
|
||||
// Content: content,
|
||||
// })
|
||||
// if err != nil {
|
||||
// glog.Errorf("消息推送失败：%v", err)
|
||||
// }
|
||||
// if resp.Code != 0 {
|
||||
// glog.Errorf("消息推送失败:%s", resp.Msg)
|
||||
// }
|
||||
//}
|
||||
func (c *Controller) PullArticle(sArticle v2.SellerArticle) error {
|
||||
skuId, _ := strconv.Atoi(sArticle.SkuID)
|
||||
resp, err := c.bidClient.LowestPrice(skuId)
|
||||
if err != nil {
|
||||
return fmt.Errorf("请求sku:%s最低价失败:%v", sArticle.SkuID, err)
|
||||
}
|
||||
|
||||
lowest, exist := lo.Find(resp.Items, func(item dw_sdk.LowestPriceItem) bool {
|
||||
return item.BiddingType == c.bidType
|
||||
})
|
||||
if !exist {
|
||||
// 没有拿到价格，将exclude设置为true，如果后面要拉取，需要手动打开
|
||||
sArticle.Exclude = true
|
||||
return c.storage.SellerArticle().Upsert(sArticle)
|
||||
}
|
||||
sell := utils.CalculateSellerPrice(append(c.seller.CalculateProcess, sArticle.CalculateProcess...), map[string]float64{
|
||||
"originalPrice": float64(lowest.LowestPrice / 100),
|
||||
})
|
||||
if sell.OriginalPrice != sArticle.Sell.OriginalPrice {
|
||||
sArticle.Sell = sell
|
||||
sArticle.HistoryPrice = append(sArticle.HistoryPrice, sArticle.Sell)
|
||||
}
|
||||
if err = c.storage.SellerArticle().Upsert(sArticle); err != nil {
|
||||
return fmt.Errorf("保存sellerArticle失败:%v", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// 修改保存状态
|
||||
func (c *Controller) setSellerStatus(status v2.SellerStatus, msg ...string) {
|
||||
c.seller.Status = status
|
||||
if len(msg) > 0 {
|
||||
c.seller.Msg = msg[0]
|
||||
}
|
||||
if err := c.storage.Seller().UpdateStatus(c.seller); err != nil {
|
||||
glog.Errorf("更新状态失败: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Controller) Ready() bool {
|
||||
return c.ready
|
||||
}
|
||||
|
||||
func (c *Controller) FetchArticlePrice(id uint) error {
|
||||
article, err := c.storage.SellerArticle().Get(storage.NewGetSellerArticleQuery().SetID(id))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = c.PullArticle(article)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// 抓取成功，利润率重新计算
|
||||
if err = c.subscribeClient.Publish(c.ctx, utils.ProfitRate_Channel, strconv.Itoa(int(article.ArticleID))); err != nil {
|
||||
glog.Warningf("发布消息失败: %v", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
15
spider/utiles.go
Normal file
15
spider/utiles.go
Normal file
@ -0,0 +1,15 @@
|
||||
package spider
|
||||
|
||||
import (
|
||||
v2 "gitea.timerzz.com/kedaya_haitao/common/structs/v2"
|
||||
dw_sdk "gitea.timerzz.com/kedaya_haitao/dw-sdk"
|
||||
)
|
||||
|
||||
var (
|
||||
BrandIdMap = map[v2.Brand]dw_sdk.BrandId{
|
||||
v2.Brand_Coach: dw_sdk.BrandId_Coach,
|
||||
}
|
||||
BrandNameMap = map[v2.Brand]dw_sdk.BrandName{
|
||||
v2.Brand_Coach: dw_sdk.BrandName_Coach,
|
||||
}
|
||||
)
|
||||
Loading…
Reference in New Issue
Block a user