From 4b9085fe954c746c8696c88c5718214c339450e0 Mon Sep 17 00:00:00 2001 From: timerzz Date: Fri, 12 Apr 2024 15:29:43 +0800 Subject: [PATCH] =?UTF-8?q?feat=20=E5=B1=95=E7=A4=BAproxy=E4=BF=A1?= =?UTF-8?q?=E6=81=AF=EF=BC=8C=E4=BF=9D=E5=AD=98pid?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main/main.go | 4 ++++ pkg/model/product.go | 2 +- pkg/options/proxy.go | 5 ++++- pkg/pools/proxy.go | 21 +++++++++++++------ pkg/watcher/coach.go | 1 + pkg/watcher/controller.go | 4 ++-- server/proxy_svc.go | 28 +++++++++++++++++++++++++ wwwroot/src/api/proxies.js | 7 +++++++ wwwroot/src/views/layout/Header.vue | 32 +++++++++++++++++++++++++---- 9 files changed, 90 insertions(+), 14 deletions(-) create mode 100644 server/proxy_svc.go create mode 100644 wwwroot/src/api/proxies.js diff --git a/main/main.go b/main/main.go index 5f00850..f14f182 100644 --- a/main/main.go +++ b/main/main.go @@ -31,6 +31,9 @@ func main() { } pool := pools.NewProxyPool(cfg.Proxy.Subscribes) + + go pool.CronUpdate(_ctx, cfg.Proxy.Interval) //定时更新代理 + pusherCtl := pusher.NewController(_ctx, db) ch := make(chan model.PushMsg, 30) pusherCtl.Consume(ch) @@ -45,6 +48,7 @@ func main() { server.NewWatcherController(watcherCtl).RegistryRouter(api) server.NewPusherSvcController(pusherCtl).RegistryRouter(api) + server.NewProxySvc(pool).RegistryRouter(api) if err = r.Listen(":2280"); err != nil { glog.Warningf("server over: %v", err) diff --git a/pkg/model/product.go b/pkg/model/product.go index bb9c326..f4981ef 100644 --- a/pkg/model/product.go +++ b/pkg/model/product.go @@ -13,7 +13,7 @@ type Product struct { UpdateErr bool `json:"updateErr"` //更新出错了 Uid string `json:"uid" gorm:"index:,unique"` - Pid string `json:"pid" gorm:"index"` //产品编号 + Pid string `json:"pid" gorm:"index:,not null"` //产品编号 Name string `json:"name"` Brand string `json:"brand"` Website WebsiteType `json:"website"` //是什么网站 diff --git a/pkg/options/proxy.go b/pkg/options/proxy.go index 8b1af31..297d27b 100644 --- a/pkg/options/proxy.go +++ b/pkg/options/proxy.go @@ -1,5 +1,8 @@ package options +import "time" + type ProxyOption struct { - Subscribes []string `yaml:"subscribes"` + Subscribes []string `yaml:"subscribes"` + Interval time.Duration `yaml:"interval"` } diff --git a/pkg/pools/proxy.go b/pkg/pools/proxy.go index 57e52f5..9ef4b5f 100644 --- a/pkg/pools/proxy.go +++ b/pkg/pools/proxy.go @@ -15,8 +15,9 @@ import ( type ProxyPool struct { m sync.Mutex - proxys proxy.ProxyList + proxies proxy.ProxyList subscribes []string //订阅url + updated time.Time } func NewProxyPool(subscribes []string) *ProxyPool { @@ -26,9 +27,13 @@ func NewProxyPool(subscribes []string) *ProxyPool { return p } +func (p *ProxyPool) Status() (proxy.ProxyList, time.Time) { + return p.proxies, p.updated +} + // Update 更新代理池 func (p *ProxyPool) Update() { - var list = make(proxy.ProxyList, 0, len(p.proxys)) + var list = make(proxy.ProxyList, 0, len(p.proxies)) for _, url := range p.subscribes { subscribeGetter, err := getter.NewSubscribe(tool.Options{"url": url}) if err != nil { @@ -40,12 +45,16 @@ func (p *ProxyPool) Update() { glog.Infof("代理源共 %d 个: %v", len(p.subscribes), p.subscribes) glog.Infof("获取代理共 %d 个", len(list)) p.m.Lock() - p.proxys = list + p.proxies = list p.m.Unlock() + p.updated = time.Now() } // CronUpdate 定时更新 func (p *ProxyPool) CronUpdate(ctx context.Context, interval time.Duration) { + if interval == 0 { + interval = time.Minute * 30 + } ticker := time.NewTicker(interval) defer ticker.Stop() for { @@ -61,12 +70,12 @@ func (p *ProxyPool) CronUpdate(ctx context.Context, interval time.Duration) { // RandomIterator 获取随机代理的迭代器 func (p *ProxyPool) RandomIterator() func() proxy.Proxy { return func() (proxy proxy.Proxy) { - if len(p.proxys) == 0 { + if len(p.proxies) == 0 { return nil } p.m.Lock() defer p.m.Unlock() - curIndex := rand.Intn(len(p.proxys)) - return p.proxys[curIndex] + curIndex := rand.Intn(len(p.proxies)) + return p.proxies[curIndex] } } diff --git a/pkg/watcher/coach.go b/pkg/watcher/coach.go index 49bab25..8d56ebb 100644 --- a/pkg/watcher/coach.go +++ b/pkg/watcher/coach.go @@ -84,6 +84,7 @@ func (c *CoachOutlet) Restart() { c.Cancel() go c.Watch() } + func (c *CoachOutlet) Watch() { c.ctx, c.cancel = context.WithCancel(c.fCtx) if err := c.db.Model(&model.Product{}).Where("uid", c.Uid()).Updates(map[string]interface{}{"watch": true, "orderable": false}).Error; err != nil { diff --git a/pkg/watcher/controller.go b/pkg/watcher/controller.go index 0651414..32c4aaa 100644 --- a/pkg/watcher/controller.go +++ b/pkg/watcher/controller.go @@ -73,8 +73,8 @@ func (c *Controller) RunWatcher(opt *options.CoachOutletOption) error { uid := opt.Uid() err := c.db.Clauses(clause.OnConflict{ Columns: []clause.Column{{Name: "uid"}}, - DoUpdates: clause.AssignmentColumns([]string{"watch", "remark", "orderable", "pusher_ids", "deleted_at"}), - }).Create(&model.Product{Uid: uid, Watch: true, Remark: opt.Remark, Orderable: false, PusherIds: opt.PusherIds}).Error + DoUpdates: clause.AssignmentColumns([]string{"pid", "watch", "remark", "orderable", "pusher_ids", "deleted_at"}), + }).Create(&model.Product{Uid: uid, Pid: opt.Pid, Watch: true, Remark: opt.Remark, Orderable: false, PusherIds: opt.PusherIds}).Error if err != nil { return err diff --git a/server/proxy_svc.go b/server/proxy_svc.go new file mode 100644 index 0000000..9cdb971 --- /dev/null +++ b/server/proxy_svc.go @@ -0,0 +1,28 @@ +package server + +import ( + "github.com/gofiber/fiber/v3" + pool "haitao_watcher/pkg/pools" +) + +type ProxySvc struct { + pool *pool.ProxyPool +} + +func NewProxySvc(pool *pool.ProxyPool) *ProxySvc { + return &ProxySvc{ + pool: pool, + } +} + +func (s *ProxySvc) RegistryRouter(r fiber.Router) { + r.Get("proxies/status", s.GetStatusInfo) +} + +func (s *ProxySvc) GetStatusInfo(ctx fiber.Ctx) error { + list, updated := s.pool.Status() + return ctx.JSON(map[string]interface{}{ + "list": list, + "updated": updated, + }) +} diff --git a/wwwroot/src/api/proxies.js b/wwwroot/src/api/proxies.js new file mode 100644 index 0000000..eaac8fd --- /dev/null +++ b/wwwroot/src/api/proxies.js @@ -0,0 +1,7 @@ +import {mande} from "mande"; + +const pushers = mande('/api/v1/proxies') + +export const getProxiesStatus = () => { + return pushers.get("/status") +} \ No newline at end of file diff --git a/wwwroot/src/views/layout/Header.vue b/wwwroot/src/views/layout/Header.vue index 703203c..b74e438 100644 --- a/wwwroot/src/views/layout/Header.vue +++ b/wwwroot/src/views/layout/Header.vue @@ -1,12 +1,36 @@