us-coach-spider/pkg/coach-client/client.go
timerzz 075a4812e4
All checks were successful
Build image / build (push) Successful in 1m30s
fix 修复coach-client接口问题
2024-05-14 17:43:08 +08:00

73 lines
1.8 KiB
Go

package coach_client
import (
"context"
"fmt"
"gitea.timerzz.com/kedaya_haitao/common/pkg/proxy"
restry_pool "gitea.timerzz.com/kedaya_haitao/common/pkg/restry-pool"
"github.com/go-resty/resty/v2"
"github.com/golang/glog"
"github.com/pkg/errors"
proxy2 "github.com/timerzz/proxypool/pkg/proxy"
"time"
)
type Client struct {
pool *proxy.ProxyPool
defaultTimeOut time.Duration
}
func NewClient(pool *proxy.ProxyPool) *Client {
return &Client{
pool: pool,
defaultTimeOut: 10 * time.Minute,
}
}
func (c *Client) ViewAllBags(ctx context.Context, page int) (resp PageDataResponse, err error) {
if page < 1 {
page = 1
}
sctx, cancel := context.WithTimeout(ctx, c.defaultTimeOut)
defer cancel()
urlPath := fmt.Sprintf("https://www.coachoutlet.com/api/get-shop/bags/view-all?page=%d", page)
err = tryRequest(sctx, urlPath, &resp, c.pool.RandomIterator())
return
}
func tryRequest(ctx context.Context, urlPath string, respData any, proxyGetter func() proxy2.Proxy) error {
for p := proxyGetter(); p != nil; p = proxyGetter() {
select {
case <-ctx.Done():
return nil
default:
}
_, err := callByProxy(ctx, p, urlPath, respData)
if err != nil {
glog.Error(err)
continue
}
return nil
}
return errors.New("未获取到可用的代理")
}
func callByProxy(ctx context.Context, p proxy2.Proxy, urlPath string, result any) (*resty.Response, error) {
addr, err := proxy.UrlToMetadata(urlPath)
if err != nil {
return nil, err
}
subCtx, cancel := context.WithTimeout(ctx, time.Second*5)
defer cancel()
conn, err := proxy.ConnFromProxy(subCtx, p, addr)
if err != nil {
return nil, errors.Wrap(err, "创建conn失败")
}
defer conn.Close()
var cli = restry_pool.GetRestyClient(conn)
defer restry_pool.PutRestyClient(cli)
return cli.R().SetResult(result).Get(urlPath)
}