73 lines
1.5 KiB
Go
73 lines
1.5 KiB
Go
|
package pools
|
|||
|
|
|
|||
|
|
import (
|
|||
|
|
"context"
|
|||
|
|
"fmt"
|
|||
|
|
"github.com/golang/glog"
|
|||
|
|
"github.com/timerzz/proxypool/pkg/getter"
|
|||
|
|
"github.com/timerzz/proxypool/pkg/proxy"
|
|||
|
|
"github.com/timerzz/proxypool/pkg/tool"
|
|||
|
|
"log/slog"
|
|||
|
|
"math/rand"
|
|||
|
|
"sync"
|
|||
|
|
"time"
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
type ProxyPool struct {
|
|||
|
|
m sync.Mutex
|
|||
|
|
proxys proxy.ProxyList
|
|||
|
|
subscribes []string //订阅url
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func NewProxyPool(subscribes []string) *ProxyPool {
|
|||
|
|
var p = &ProxyPool{}
|
|||
|
|
p.subscribes = subscribes
|
|||
|
|
p.Update()
|
|||
|
|
return p
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// Update 更新代理池
|
|||
|
|
func (p *ProxyPool) Update() {
|
|||
|
|
var list = make(proxy.ProxyList, 0, len(p.proxys))
|
|||
|
|
for _, url := range p.subscribes {
|
|||
|
|
subscribeGetter, err := getter.NewSubscribe(tool.Options{"url": url})
|
|||
|
|
if err != nil {
|
|||
|
|
slog.Warn(fmt.Sprintf("创建Subscribe Getter失败：%v", err))
|
|||
|
|
continue
|
|||
|
|
}
|
|||
|
|
list = list.UniqAppendProxyList(subscribeGetter.Get())
|
|||
|
|
}
|
|||
|
|
glog.Infof("代理源共 %d 个: %v", len(p.subscribes), p.subscribes)
|
|||
|
|
glog.Infof("获取代理共 %d 个", len(list))
|
|||
|
|
p.m.Lock()
|
|||
|
|
p.proxys = list
|
|||
|
|
p.m.Unlock()
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// CronUpdate 定时更新
|
|||
|
|
func (p *ProxyPool) CronUpdate(ctx context.Context, interval time.Duration) {
|
|||
|
|
ticker := time.NewTicker(interval)
|
|||
|
|
defer ticker.Stop()
|
|||
|
|
for {
|
|||
|
|
select {
|
|||
|
|
case <-ctx.Done():
|
|||
|
|
return
|
|||
|
|
case <-ticker.C:
|
|||
|
|
p.Update()
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// RandomIterator 获取随机代理的迭代器
|
|||
|
|
func (p *ProxyPool) RandomIterator() func() proxy.Proxy {
|
|||
|
|
return func() (proxy proxy.Proxy) {
|
|||
|
|
if len(p.proxys) == 0 {
|
|||
|
|
return nil
|
|||
|
|
}
|
|||
|
|
p.m.Lock()
|
|||
|
|
defer p.m.Unlock()
|
|||
|
|
curIndex := rand.Intn(len(p.proxys))
|
|||
|
|
return p.proxys[curIndex]
|
|||
|
|
}
|
|||
|
|
}
|