us-coach-spider/pkg/coach-client/client.go

package coach_client

import (
	"context"
	"fmt"
	"gitea.timerzz.com/kedaya_haitao/common/pkg/proxy"
	restry_pool "gitea.timerzz.com/kedaya_haitao/common/pkg/restry-pool"
	"github.com/go-resty/resty/v2"
	"github.com/golang/glog"
	"github.com/pkg/errors"
	proxy2 "github.com/timerzz/proxypool/pkg/proxy"
	"time"
)

type Client struct {
	pool           *proxy.ProxyPool
	defaultTimeOut time.Duration
}

func NewClient(pool *proxy.ProxyPool) *Client {
	return &Client{
		pool:           pool,
		defaultTimeOut: 10 * time.Minute,
	}
}

func (c *Client) ViewAllBags(ctx context.Context, page int) (resp PageDataResponse, err error) {
	if page < 1 {
		page = 1
	}
	sctx, cancel := context.WithTimeout(ctx, c.defaultTimeOut)
	defer cancel()
	urlPath := fmt.Sprintf("https://www.coachoutlet.com/api/get-shop/bags/view-all?page=%d", page)
	err = tryRequest(sctx, urlPath, &resp, c.pool.RandomIterator())
	return
}

func tryRequest(ctx context.Context, urlPath string, respData any, proxyGetter func() proxy2.Proxy) error {
	for p := proxyGetter(); p != nil; p = proxyGetter() {
		select {
		case <-ctx.Done():
			return nil
		default:

		}
		_, err := callByProxy(ctx, p, urlPath, respData)
		if err != nil {
			glog.Error(err)
			continue
		}
		return nil
	}
	return errors.New("未获取到可用的代理")
}

func callByProxy(ctx context.Context, p proxy2.Proxy, urlPath string, result any) (*resty.Response, error) {
	addr, err := proxy.UrlToMetadata(urlPath)
	if err != nil {
		return nil, err
	}
	subCtx, cancel := context.WithTimeout(ctx, time.Second*5)
	defer cancel()
	conn, err := proxy.ConnFromProxy(subCtx, p, addr)
	if err != nil {
		return nil, errors.Wrap(err, "创建conn失败")
	}
	defer conn.Close()

	var cli = restry_pool.GetRestyClient(conn)
	defer restry_pool.PutRestyClient(cli)
	return cli.R().SetResult(result).Get(urlPath)
}