feat 抽出coach请求

2024-05-21 15:38:09 +08:00 · 2024-05-21 15:38:09 +08:00 · 8f70b150b0
commit 8f70b150b0
parent 3202eb827b
7 changed files with 8 additions and 152 deletions
--- a/cmd/main.go
+++ b/cmd/main.go
@ -2,10 +2,10 @@ package main

 import (
 	"context"
-	coach_client "gitea.timerzz.com/kedaya_haitao/coach-spider/pkg/coach-client"
 	"gitea.timerzz.com/kedaya_haitao/coach-spider/pkg/options"
 	"gitea.timerzz.com/kedaya_haitao/coach-spider/product"
 	"gitea.timerzz.com/kedaya_haitao/coach-spider/server"
+	"gitea.timerzz.com/kedaya_haitao/common/pkg/coach-client"
 	"gitea.timerzz.com/kedaya_haitao/common/pkg/database"
 	"gitea.timerzz.com/kedaya_haitao/common/pkg/proxy"
 	"github.com/gofiber/fiber/v3"
--- a/go.mod
+++ b/go.mod
@ -5,13 +5,10 @@ go 1.22.2
 toolchain go1.22.3

 require (
-	gitea.timerzz.com/kedaya_haitao/common v0.0.0-20240515114301-570e19d76c1a
-	github.com/go-resty/resty/v2 v2.13.1
+	gitea.timerzz.com/kedaya_haitao/common v0.0.0-20240521073651-0653521e73ea
 	github.com/gofiber/fiber/v3 v3.0.0-beta.2
 	github.com/golang/glog v1.2.1
-	github.com/pkg/errors v0.9.1
 	github.com/samber/lo v1.39.0
-	github.com/timerzz/proxypool v0.0.0-20240512142241-f74bcb3534c5
 	gopkg.in/yaml.v3 v3.0.1
 	gorm.io/gorm v1.25.10
 )
@ -42,6 +39,7 @@ require (
 	github.com/ericlagergren/subtle v0.0.0-20220507045147-890d697da010 // indirect
 	github.com/gaukas/godicttls v0.0.4 // indirect
 	github.com/go-ole/go-ole v1.3.0 // indirect
+	github.com/go-resty/resty/v2 v2.13.1 // indirect
 	github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
 	github.com/gobwas/glob v0.2.3 // indirect
 	github.com/gobwas/httphead v0.1.0 // indirect
@ -96,6 +94,7 @@ require (
 	github.com/openacid/low v0.1.21 // indirect
 	github.com/oschwald/maxminddb-golang v1.12.0 // indirect
 	github.com/pierrec/lz4/v4 v4.1.14 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
 	github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
 	github.com/puzpuzpuz/xsync/v3 v3.1.0 // indirect
 	github.com/quic-go/qpack v0.4.0 // indirect
@ -116,6 +115,7 @@ require (
 	github.com/sina-ghaderi/rabbitio v0.0.0-20220730151941-9ce26f4f872e // indirect
 	github.com/sirupsen/logrus v1.9.3 // indirect
 	github.com/temoto/robotstxt v1.1.2 // indirect
+	github.com/timerzz/proxypool v0.0.0-20240512142241-f74bcb3534c5 // indirect
 	github.com/tklauser/go-sysconf v0.3.12 // indirect
 	github.com/tklauser/numcpus v0.6.1 // indirect
 	github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
--- a/go.sum
+++ b/go.sum
@ -1,8 +1,6 @@
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
-gitea.timerzz.com/kedaya_haitao/common v0.0.0-20240515055453-202f8a89111a h1:D1yaUoTNJ1GyPzrVzrqFr8MBrOS/IpfyMg5kjE8diwk=
-gitea.timerzz.com/kedaya_haitao/common v0.0.0-20240515055453-202f8a89111a/go.mod h1:cgLy4gB0z2UZD3kBp1IAemt28dkaKNdt4RC0LNkE1I8=
-gitea.timerzz.com/kedaya_haitao/common v0.0.0-20240515114301-570e19d76c1a h1:lmhF73Y8SSwNntZzOFwVrSP2Y0UOh9l8anHr6lBQ/xI=
-gitea.timerzz.com/kedaya_haitao/common v0.0.0-20240515114301-570e19d76c1a/go.mod h1:cgLy4gB0z2UZD3kBp1IAemt28dkaKNdt4RC0LNkE1I8=
+gitea.timerzz.com/kedaya_haitao/common v0.0.0-20240521073651-0653521e73ea h1:asjEGs0rxu+pMXEd6bWipoingfhLBG1qP3Gt6lSy2Kg=
+gitea.timerzz.com/kedaya_haitao/common v0.0.0-20240521073651-0653521e73ea/go.mod h1:cgLy4gB0z2UZD3kBp1IAemt28dkaKNdt4RC0LNkE1I8=
 github.com/3andne/restls-client-go v0.1.6 h1:tRx/YilqW7iHpgmEL4E1D8dAsuB0tFF3uvncS+B6I08=
 github.com/3andne/restls-client-go v0.1.6/go.mod h1:iEdTZNt9kzPIxjIGSMScUFSBrUH6bFRNg0BWlP4orEY=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
--- a/pkg/coach-client/client.go
+++ b/pkg/coach-client/client.go
@ -1,72 +0,0 @@
-package coach_client
-
-import (
-	"context"
-	"fmt"
-	"gitea.timerzz.com/kedaya_haitao/common/pkg/proxy"
-	restry_pool "gitea.timerzz.com/kedaya_haitao/common/pkg/restry-pool"
-	"github.com/go-resty/resty/v2"
-	"github.com/pkg/errors"
-	proxy2 "github.com/timerzz/proxypool/pkg/proxy"
-	"log/slog"
-	"time"
-)
-
-type Client struct {
-	pool           *proxy.ProxyPool
-	defaultTimeOut time.Duration
-}
-
-func NewClient(pool *proxy.ProxyPool) *Client {
-	return &Client{
-		pool:           pool,
-		defaultTimeOut: 10 * time.Minute,
-	}
-}
-
-func (c *Client) ViewAllBags(ctx context.Context, page int) (resp PageDataResponse, err error) {
-	if page < 1 {
-		page = 1
-	}
-	sctx, cancel := context.WithTimeout(ctx, c.defaultTimeOut)
-	defer cancel()
-	urlPath := fmt.Sprintf("https://www.coachoutlet.com/api/get-shop/bags/view-all?page=%d", page)
-	err = tryRequest(sctx, urlPath, &resp, c.pool.RandomIterator())
-	return
-}
-
-func tryRequest(ctx context.Context, urlPath string, respData any, proxyGetter func() proxy2.Proxy) error {
-	for p := proxyGetter(); p != nil; p = proxyGetter() {
-		select {
-		case <-ctx.Done():
-			return nil
-		default:
-
-		}
-		_, err := callByProxy(ctx, p, urlPath, respData)
-		if err != nil {
-			slog.Debug(err.Error())
-			continue
-		}
-		return nil
-	}
-	return errors.New("未获取到可用的代理")
-}
-
-func callByProxy(ctx context.Context, p proxy2.Proxy, urlPath string, result any) (*resty.Response, error) {
-	addr, err := proxy.UrlToMetadata(urlPath)
-	if err != nil {
-		return nil, err
-	}
-	subCtx, cancel := context.WithTimeout(ctx, time.Second*5)
-	defer cancel()
-	conn, err := proxy.ConnFromProxy(subCtx, p, addr)
-	if err != nil {
-		return nil, errors.Wrap(err, "创建conn失败")
-	}
-	defer conn.Close()
-
-	var cli = restry_pool.GetRestyClient(conn)
-	defer restry_pool.PutRestyClient(cli)
-	return cli.R().SetResult(result).Get(urlPath)
-}
--- a/pkg/coach-client/client_test.go
+++ b/pkg/coach-client/client_test.go
@ -1,18 +0,0 @@
-package coach_client
-
-import (
-	"context"
-	"gitea.timerzz.com/kedaya_haitao/common/pkg/proxy"
-	"testing"
-)
-
-func TestClient(t *testing.T) {
-	subs := []string{"https://us.timerzz.com:26106/vmess/sub", "https://us.timerzz.com:26106/trojan/sub", "https://us.timerzz.com:26106/ssr/sub"}
-	pool := proxy.NewProxyPool(subs)
-	client := NewClient(pool)
-	resp, err := client.ViewAllBags(context.Background(), 1)
-	if err != nil {
-		t.Fatal(err)
-	}
-	t.Log(resp)
-}
--- a/pkg/coach-client/model.go
+++ b/pkg/coach-client/model.go
@ -1,52 +0,0 @@
-package coach_client
-
-type PageDataResponse struct {
-	PageData struct {
-		Total      int       `json:"total"`
-		Page       int       `json:"page"`
-		TotalPages int       `json:"totalPages"`
-		PageSize   int       `json:"pageSize"`
-		Products   []Product `json:"products"`
-	} `json:"pageData"`
-}
-type Product struct {
-	Name           string    `json:"name"`
-	Colors         []Color   `json:"colors"`
-	VariantsOnSale []Variant `json:"variantsOnSale"`
-}
-
-type Color struct {
-	Id        string `json:"id"`   //"id": "IMDQC",
-	Text      string `json:"text"` //颜色的描述
-	Orderable bool   `json:"orderable"`
-	Media     Media  `json:"media"`
-	Url       string `json:"url"`      //"/products/eliza-flap-crossbody-bag-in-signature-canvas/CP009-IMDQC.html",
-	VgId      string `json:"vgId"`     // "vgId": "CP009-IMDQC",
-	MasterId  string `json:"masterId"` //"masterId": "CP009"
-}
-
-type Media struct {
-	Thumbnail Thumbnail `json:"thumbnail"`
-}
-
-type Thumbnail struct {
-	Src string `json:"src"`
-}
-
-type Variant struct {
-	Id     string `json:"id"`
-	OnSale bool   `json:"onSale"`
-	Price  Price  `json:"price"`
-}
-
-type Price struct {
-	Sales               Sales `json:"sales"`
-	MarkdownDiscPercent int   `json:"markdownDiscPercent"`
-}
-
-type Sales struct {
-	Value        float64 `json:"value"`
-	Currency     string  `json:"currency"`
-	Formatted    string  `json:"formatted"`
-	DecimalPrice string  `json:"decimalPrice"`
-}
--- a/product/controller.go
+++ b/product/controller.go
@ -3,8 +3,8 @@ package product
 import (
 	"context"
 	"fmt"
-	coach_client "gitea.timerzz.com/kedaya_haitao/coach-spider/pkg/coach-client"
 	"gitea.timerzz.com/kedaya_haitao/common/model/product"
+	coach_client "gitea.timerzz.com/kedaya_haitao/common/pkg/coach-client"
 	"github.com/golang/glog"
 	"github.com/samber/lo"
 	"gorm.io/gorm"