From 9a916cc85e5230b40403fe9ea15d4f3f1a0a6fb7 Mon Sep 17 00:00:00 2001 From: timerzz Date: Tue, 14 May 2024 21:18:24 +0800 Subject: [PATCH] =?UTF-8?q?feat=20=E6=B7=BB=E5=8A=A0=E4=BF=AE=E6=94=B9spid?= =?UTF-8?q?er=20=E9=85=8D=E7=BD=AE=E7=9A=84=E6=96=B9=E5=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cmd/main.go | 31 ++++++++++++++++++------------- go.mod | 12 +++++++++--- go.sum | 24 ++++++++++++++++++------ product/option.go | 24 +++++++++++++++++++++--- server/spider.go | 35 +++++++++++++++++++++++++++++++++++ 5 files changed, 101 insertions(+), 25 deletions(-) create mode 100644 server/spider.go diff --git a/cmd/main.go b/cmd/main.go index 0e52492..0b6ba53 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -5,8 +5,11 @@ import ( coach_client "gitea.timerzz.com/kedaya_haitao/coach-spider/pkg/coach-client" "gitea.timerzz.com/kedaya_haitao/coach-spider/pkg/options" "gitea.timerzz.com/kedaya_haitao/coach-spider/product" + "gitea.timerzz.com/kedaya_haitao/coach-spider/server" "gitea.timerzz.com/kedaya_haitao/common/pkg/database" "gitea.timerzz.com/kedaya_haitao/common/pkg/proxy" + "github.com/gofiber/fiber/v3" + "github.com/gofiber/fiber/v3/middleware/cors" "github.com/golang/glog" "log/slog" "os" @@ -36,22 +39,24 @@ func main() { productCtl := product.NewController(client, db) - //r := fiber.New() - //r.Use(cors.New()) - // - //r.Get("/health", func(ctx fiber.Ctx) error { - // return ctx.SendString("ok") - //}) - // - //api := r.Group("/api/v1") + go productCtl.Run(ctx) + + r := fiber.New() + r.Use(cors.New()) + + r.Get("/health", func(ctx fiber.Ctx) error { + return ctx.SendString("ok") + }) + + api := r.Group("/api/v1") + + server.NewSpiderSvc(productCtl).RegistryRouter(api) // //server.NewWatcherController(watcherCtl).RegistryRouter(api) //server.NewPusherSvcController(pusherCtl).RegistryRouter(api) //server.NewProxySvc(pool).RegistryRouter(api) - //if err = r.Listen(":8080"); err != nil { - // glog.Warningf("server over: %v", err) - //} - - productCtl.Run(ctx) + if err = r.Listen(":8080"); err != nil { + glog.Warningf("server over: %v", err) + } } diff --git a/go.mod b/go.mod index 1c050d7..5e002bb 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.22.2 require ( gitea.timerzz.com/kedaya_haitao/common v0.0.0-20240514123648-99c3e85dbc18 github.com/go-resty/resty/v2 v2.13.1 + github.com/gofiber/fiber/v3 v3.0.0-beta.2 github.com/golang/glog v1.2.1 github.com/pkg/errors v0.9.1 github.com/samber/lo v1.39.0 @@ -19,7 +20,7 @@ require ( github.com/RyuaNerin/go-krypto v1.2.4 // indirect github.com/Yawning/aez v0.0.0-20211027044916-e49e68abd344 // indirect github.com/aead/chacha20 v0.0.0-20180709150244-8b13a72661da // indirect - github.com/andybalholm/brotli v1.0.6 // indirect + github.com/andybalholm/brotli v1.1.0 // indirect github.com/andybalholm/cascadia v1.3.2 // indirect github.com/antchfx/htmlquery v1.3.1 // indirect github.com/antchfx/xmlquery v1.3.15 // indirect @@ -45,12 +46,14 @@ require ( github.com/gobwas/pool v0.2.1 // indirect github.com/gobwas/ws v1.3.2 // indirect github.com/gocolly/colly/v2 v2.1.0 // indirect + github.com/gofiber/utils/v2 v2.0.0-beta.4 // indirect github.com/gofrs/uuid/v5 v5.1.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/google/btree v1.1.2 // indirect github.com/google/go-cmp v0.6.0 // indirect github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 // indirect + github.com/google/uuid v1.6.0 // indirect github.com/hashicorp/yamux v0.1.1 // indirect github.com/insomniacslk/dhcp v0.0.0-20240419123447-f1cffa2c0c49 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect @@ -61,13 +64,13 @@ require ( github.com/josharian/native v1.1.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/kennygrant/sanitize v1.2.4 // indirect - github.com/klauspost/compress v1.17.4 // indirect + github.com/klauspost/compress v1.17.6 // indirect github.com/klauspost/cpuid/v2 v2.2.7 // indirect github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/lunixbochs/struc v0.0.0-20200707160740-784aaebc1d40 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect - github.com/mattn/go-isatty v0.0.19 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect github.com/mdlayher/netlink v1.7.2 // indirect github.com/mdlayher/socket v0.4.1 // indirect github.com/metacubex/gopacket v1.1.20-0.20230608035415-7e2f98a3e759 // indirect @@ -115,6 +118,9 @@ require ( github.com/tklauser/numcpus v0.6.1 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/u-root/uio v0.0.0-20230220225925-ffce2a382923 // indirect + github.com/valyala/bytebufferpool v1.0.0 // indirect + github.com/valyala/fasthttp v1.52.0 // indirect + github.com/valyala/tcplisten v1.0.0 // indirect github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect github.com/x-cray/logrus-prefixed-formatter v0.5.2 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect diff --git a/go.sum b/go.sum index a947563..312dd2e 100644 --- a/go.sum +++ b/go.sum @@ -14,8 +14,8 @@ github.com/Yawning/aez v0.0.0-20211027044916-e49e68abd344 h1:cDVUiFo+npB0ZASqnw4 github.com/Yawning/aez v0.0.0-20211027044916-e49e68abd344/go.mod h1:9pIqrY6SXNL8vjRQE5Hd/OL5GyK/9MrGUWs87z/eFfk= github.com/aead/chacha20 v0.0.0-20180709150244-8b13a72661da h1:KjTM2ks9d14ZYCvmHS9iAKVt9AyzRSqNU1qabPih5BY= github.com/aead/chacha20 v0.0.0-20180709150244-8b13a72661da/go.mod h1:eHEWzANqSiWQsof+nXEI9bUVUyV6F53Fp89EuCh2EAA= -github.com/andybalholm/brotli v1.0.6 h1:Yf9fFpf49Zrxb9NlQaluyE92/+X7UVHlhMNJN2sxfOI= -github.com/andybalholm/brotli v1.0.6/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= +github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M= +github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY= github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= github.com/andybalholm/cascadia v1.2.0/go.mod h1:YCyR8vOZT9aZ1CHEd8ap0gMVm2aFgxBp0T0eFw1RUQY= github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= @@ -95,6 +95,10 @@ github.com/gobwas/ws v1.3.2/go.mod h1:hRKAFb8wOxFROYNsT1bqfWnhX+b5MFeJM9r2ZSwg/K github.com/gocolly/colly v1.2.0/go.mod h1:Hof5T3ZswNVsOHYmba1u03W65HDWgpV5HifSuueE0EA= github.com/gocolly/colly/v2 v2.1.0 h1:k0DuZkDoCsx51bKpRJNEmcxcp+W5N8ziuwGaSDuFoGs= github.com/gocolly/colly/v2 v2.1.0/go.mod h1:I2MuhsLjQ+Ex+IzK3afNS8/1qP3AedHOusRPcRdC5o0= +github.com/gofiber/fiber/v3 v3.0.0-beta.2 h1:mVVgt8PTaHGup3NGl/+7U7nEoZaXJ5OComV4E+HpAao= +github.com/gofiber/fiber/v3 v3.0.0-beta.2/go.mod h1:w7sdfTY0okjZ1oVH6rSOGvuACUIt0By1iK0HKUb3uqM= +github.com/gofiber/utils/v2 v2.0.0-beta.4 h1:1gjbVFFwVwUb9arPcqiB6iEjHBwo7cHsyS41NeIW3co= +github.com/gofiber/utils/v2 v2.0.0-beta.4/go.mod h1:sdRsPU1FXX6YiDGGxd+q2aPJRMzpsxdzCXo9dz+xtOY= github.com/gofrs/uuid/v5 v5.1.0 h1:S5rqVKIigghZTCBKPCw0Y+bXkn26K3TB5mvQq2Ix8dk= github.com/gofrs/uuid/v5 v5.1.0/go.mod h1:CDOjlDMVAtN56jqyRUZh58JT31Tiw7/oQyEXZV+9bD8= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= @@ -133,6 +137,8 @@ github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 h1:Xim43kblpZXfIBQsbu github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo= github.com/google/tink/go v1.6.1 h1:t7JHqO8Ath2w2ig5vjwQYJzhGEZymedQc90lQXUBa4I= github.com/google/tink/go v1.6.1/go.mod h1:IGW53kTgag+st5yPhKKwJ6u2l+SSp5/v9XF7spovjlY= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/hashicorp/yamux v0.1.1 h1:yrQxtgseBDrq9Y652vSRDvsKCJKOUD+GzTS4Y0Y8pvE= github.com/hashicorp/yamux v0.1.1/go.mod h1:CtWFDAQgb7dxtzFs4tWbplKIe2jSi3+5vKbgIO0SLnQ= github.com/insomniacslk/dhcp v0.0.0-20240419123447-f1cffa2c0c49 h1:/OuvSMGT9+xnyZ+7MZQ1zdngaCCAdPoSw8B/uurZ7pg= @@ -156,8 +162,8 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/kennygrant/sanitize v1.2.4 h1:gN25/otpP5vAsO2djbMhF/LQX6R7+O1TB4yv8NzpJ3o= github.com/kennygrant/sanitize v1.2.4/go.mod h1:LGsjYYtgxbetdg5owWB2mpgUL6e2nfw2eObZ0u0qvak= -github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4= -github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= +github.com/klauspost/compress v1.17.6 h1:60eq2E/jlfwQXtvZEeBUYADs+BwKBWURIY+Gj2eRGjI= +github.com/klauspost/compress v1.17.6/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM= github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= @@ -175,8 +181,8 @@ github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJ github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= -github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mdlayher/netlink v1.7.2 h1:/UtM3ofJap7Vl4QWCPDGXY8d3GIY2UGSDbK+QWmY8/g= github.com/mdlayher/netlink v1.7.2/go.mod h1:xraEF7uJbxLhc5fpHL4cPe221LI2bdttWlU+ZGLfQSw= github.com/mdlayher/socket v0.4.1 h1:eM9y2/jlbs1M615oshPQOHZzj6R6wMT7bX5NPiQvn2U= @@ -309,6 +315,12 @@ github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= github.com/u-root/uio v0.0.0-20230220225925-ffce2a382923 h1:tHNk7XK9GkmKUR6Gh8gVBKXc2MVSZ4G/NnWLtzw4gNA= github.com/u-root/uio v0.0.0-20230220225925-ffce2a382923/go.mod h1:eLL9Nub3yfAho7qB0MzZizFhTU2QkLeoVsWdHtDW264= +github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasthttp v1.52.0 h1:wqBQpxH71XW0e2g+Og4dzQM8pk34aFYlA1Ga8db7gU0= +github.com/valyala/fasthttp v1.52.0/go.mod h1:hf5C4QnVMkNXMspnsUlfM3WitlgYflyhHYoKol/szxQ= +github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8= +github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc= github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= diff --git a/product/option.go b/product/option.go index 4d8646d..0b641ee 100644 --- a/product/option.go +++ b/product/option.go @@ -4,11 +4,29 @@ import "time" type Option struct { ID uint `gorm:"primary_key" json:"id"` - Interval time.Duration `yaml:"interval"` - ExchangeRate float64 `yaml:"exchangeRate"` //汇率 - Freight float64 `yaml:"freight"` //运费 + Interval time.Duration `yaml:"interval" json:"interval"` + ExchangeRate float64 `yaml:"exchangeRate" json:"exchangeRate"` //汇率 + Freight float64 `yaml:"freight" json:"freight"` //运费 } func (c *Controller) LoadOption() { c.db.FirstOrCreate(&c.Option, Option{Interval: time.Hour * 30, ExchangeRate: 7.3, Freight: 100}) } + +func (c *Controller) SaveOption(opt Option) { + if opt.Interval > 0 { + c.Option.Interval = opt.Interval + } + if opt.ExchangeRate > 0 { + c.Option.ExchangeRate = opt.ExchangeRate + } + if opt.Freight > 0 { + c.Option.Freight = opt.Freight + } + opt.ID = 1 + c.db.Updates(opt) +} + +func (c *Controller) GetOption() Option { + return c.Option +} diff --git a/server/spider.go b/server/spider.go new file mode 100644 index 0000000..955615e --- /dev/null +++ b/server/spider.go @@ -0,0 +1,35 @@ +package server + +import ( + "gitea.timerzz.com/kedaya_haitao/coach-spider/product" + "github.com/gofiber/fiber/v3" +) + +type SpiderSvc struct { + ctl *product.Controller +} + +func NewSpiderSvc(ctl *product.Controller) *SpiderSvc { + return &SpiderSvc{ + ctl: ctl, + } +} + +func (s *SpiderSvc) RegistryRouter(r fiber.Router) { + r.Get("spider/cfg", s.GetSpiderCfg) + r.Post("spider/cfg", s.SetSpiderCfg) +} + +func (s *SpiderSvc) GetSpiderCfg(ctx fiber.Ctx) error { + opt := s.ctl.GetOption() + return ctx.JSON(&opt) +} + +func (s *SpiderSvc) SetSpiderCfg(ctx fiber.Ctx) error { + var opt product.Option + if err := ctx.Bind().JSON(&opt); err != nil { + return err + } + s.ctl.SaveOption(opt) + return nil +}