diff --git a/cmd/main.go b/cmd/main.go index 72a7457..2304c39 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -2,10 +2,10 @@ package main import ( "context" - coach_client "gitea.timerzz.com/kedaya_haitao/coach-spider/pkg/coach-client" "gitea.timerzz.com/kedaya_haitao/coach-spider/pkg/options" "gitea.timerzz.com/kedaya_haitao/coach-spider/product" "gitea.timerzz.com/kedaya_haitao/coach-spider/server" + "gitea.timerzz.com/kedaya_haitao/common/pkg/coach-client" "gitea.timerzz.com/kedaya_haitao/common/pkg/database" "gitea.timerzz.com/kedaya_haitao/common/pkg/proxy" "github.com/gofiber/fiber/v3" diff --git a/go.mod b/go.mod index 5714ee1..2ba9944 100644 --- a/go.mod +++ b/go.mod @@ -5,13 +5,10 @@ go 1.22.2 toolchain go1.22.3 require ( - gitea.timerzz.com/kedaya_haitao/common v0.0.0-20240515114301-570e19d76c1a - github.com/go-resty/resty/v2 v2.13.1 + gitea.timerzz.com/kedaya_haitao/common v0.0.0-20240521073651-0653521e73ea github.com/gofiber/fiber/v3 v3.0.0-beta.2 github.com/golang/glog v1.2.1 - github.com/pkg/errors v0.9.1 github.com/samber/lo v1.39.0 - github.com/timerzz/proxypool v0.0.0-20240512142241-f74bcb3534c5 gopkg.in/yaml.v3 v3.0.1 gorm.io/gorm v1.25.10 ) @@ -42,6 +39,7 @@ require ( github.com/ericlagergren/subtle v0.0.0-20220507045147-890d697da010 // indirect github.com/gaukas/godicttls v0.0.4 // indirect github.com/go-ole/go-ole v1.3.0 // indirect + github.com/go-resty/resty/v2 v2.13.1 // indirect github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect github.com/gobwas/glob v0.2.3 // indirect github.com/gobwas/httphead v0.1.0 // indirect @@ -96,6 +94,7 @@ require ( github.com/openacid/low v0.1.21 // indirect github.com/oschwald/maxminddb-golang v1.12.0 // indirect github.com/pierrec/lz4/v4 v4.1.14 // indirect + github.com/pkg/errors v0.9.1 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect github.com/puzpuzpuz/xsync/v3 v3.1.0 // indirect github.com/quic-go/qpack v0.4.0 // indirect @@ -116,6 +115,7 @@ require ( github.com/sina-ghaderi/rabbitio v0.0.0-20220730151941-9ce26f4f872e // indirect github.com/sirupsen/logrus v1.9.3 // indirect github.com/temoto/robotstxt v1.1.2 // indirect + github.com/timerzz/proxypool v0.0.0-20240512142241-f74bcb3534c5 // indirect github.com/tklauser/go-sysconf v0.3.12 // indirect github.com/tklauser/numcpus v0.6.1 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect diff --git a/go.sum b/go.sum index 8298439..9728775 100644 --- a/go.sum +++ b/go.sum @@ -1,8 +1,6 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -gitea.timerzz.com/kedaya_haitao/common v0.0.0-20240515055453-202f8a89111a h1:D1yaUoTNJ1GyPzrVzrqFr8MBrOS/IpfyMg5kjE8diwk= -gitea.timerzz.com/kedaya_haitao/common v0.0.0-20240515055453-202f8a89111a/go.mod h1:cgLy4gB0z2UZD3kBp1IAemt28dkaKNdt4RC0LNkE1I8= -gitea.timerzz.com/kedaya_haitao/common v0.0.0-20240515114301-570e19d76c1a h1:lmhF73Y8SSwNntZzOFwVrSP2Y0UOh9l8anHr6lBQ/xI= -gitea.timerzz.com/kedaya_haitao/common v0.0.0-20240515114301-570e19d76c1a/go.mod h1:cgLy4gB0z2UZD3kBp1IAemt28dkaKNdt4RC0LNkE1I8= +gitea.timerzz.com/kedaya_haitao/common v0.0.0-20240521073651-0653521e73ea h1:asjEGs0rxu+pMXEd6bWipoingfhLBG1qP3Gt6lSy2Kg= +gitea.timerzz.com/kedaya_haitao/common v0.0.0-20240521073651-0653521e73ea/go.mod h1:cgLy4gB0z2UZD3kBp1IAemt28dkaKNdt4RC0LNkE1I8= github.com/3andne/restls-client-go v0.1.6 h1:tRx/YilqW7iHpgmEL4E1D8dAsuB0tFF3uvncS+B6I08= github.com/3andne/restls-client-go v0.1.6/go.mod h1:iEdTZNt9kzPIxjIGSMScUFSBrUH6bFRNg0BWlP4orEY= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= diff --git a/pkg/coach-client/client.go b/pkg/coach-client/client.go deleted file mode 100644 index c847417..0000000 --- a/pkg/coach-client/client.go +++ /dev/null @@ -1,72 +0,0 @@ -package coach_client - -import ( - "context" - "fmt" - "gitea.timerzz.com/kedaya_haitao/common/pkg/proxy" - restry_pool "gitea.timerzz.com/kedaya_haitao/common/pkg/restry-pool" - "github.com/go-resty/resty/v2" - "github.com/pkg/errors" - proxy2 "github.com/timerzz/proxypool/pkg/proxy" - "log/slog" - "time" -) - -type Client struct { - pool *proxy.ProxyPool - defaultTimeOut time.Duration -} - -func NewClient(pool *proxy.ProxyPool) *Client { - return &Client{ - pool: pool, - defaultTimeOut: 10 * time.Minute, - } -} - -func (c *Client) ViewAllBags(ctx context.Context, page int) (resp PageDataResponse, err error) { - if page < 1 { - page = 1 - } - sctx, cancel := context.WithTimeout(ctx, c.defaultTimeOut) - defer cancel() - urlPath := fmt.Sprintf("https://www.coachoutlet.com/api/get-shop/bags/view-all?page=%d", page) - err = tryRequest(sctx, urlPath, &resp, c.pool.RandomIterator()) - return -} - -func tryRequest(ctx context.Context, urlPath string, respData any, proxyGetter func() proxy2.Proxy) error { - for p := proxyGetter(); p != nil; p = proxyGetter() { - select { - case <-ctx.Done(): - return nil - default: - - } - _, err := callByProxy(ctx, p, urlPath, respData) - if err != nil { - slog.Debug(err.Error()) - continue - } - return nil - } - return errors.New("未获取到可用的代理") -} - -func callByProxy(ctx context.Context, p proxy2.Proxy, urlPath string, result any) (*resty.Response, error) { - addr, err := proxy.UrlToMetadata(urlPath) - if err != nil { - return nil, err - } - subCtx, cancel := context.WithTimeout(ctx, time.Second*5) - defer cancel() - conn, err := proxy.ConnFromProxy(subCtx, p, addr) - if err != nil { - return nil, errors.Wrap(err, "创建conn失败") - } - defer conn.Close() - - var cli = restry_pool.GetRestyClient(conn) - defer restry_pool.PutRestyClient(cli) - return cli.R().SetResult(result).Get(urlPath) -} diff --git a/pkg/coach-client/client_test.go b/pkg/coach-client/client_test.go deleted file mode 100644 index a8eff65..0000000 --- a/pkg/coach-client/client_test.go +++ /dev/null @@ -1,18 +0,0 @@ -package coach_client - -import ( - "context" - "gitea.timerzz.com/kedaya_haitao/common/pkg/proxy" - "testing" -) - -func TestClient(t *testing.T) { - subs := []string{"https://us.timerzz.com:26106/vmess/sub", "https://us.timerzz.com:26106/trojan/sub", "https://us.timerzz.com:26106/ssr/sub"} - pool := proxy.NewProxyPool(subs) - client := NewClient(pool) - resp, err := client.ViewAllBags(context.Background(), 1) - if err != nil { - t.Fatal(err) - } - t.Log(resp) -} diff --git a/pkg/coach-client/model.go b/pkg/coach-client/model.go deleted file mode 100644 index e5fb70a..0000000 --- a/pkg/coach-client/model.go +++ /dev/null @@ -1,52 +0,0 @@ -package coach_client - -type PageDataResponse struct { - PageData struct { - Total int `json:"total"` - Page int `json:"page"` - TotalPages int `json:"totalPages"` - PageSize int `json:"pageSize"` - Products []Product `json:"products"` - } `json:"pageData"` -} -type Product struct { - Name string `json:"name"` - Colors []Color `json:"colors"` - VariantsOnSale []Variant `json:"variantsOnSale"` -} - -type Color struct { - Id string `json:"id"` //"id": "IMDQC", - Text string `json:"text"` //颜色的描述 - Orderable bool `json:"orderable"` - Media Media `json:"media"` - Url string `json:"url"` //"/products/eliza-flap-crossbody-bag-in-signature-canvas/CP009-IMDQC.html", - VgId string `json:"vgId"` // "vgId": "CP009-IMDQC", - MasterId string `json:"masterId"` //"masterId": "CP009" -} - -type Media struct { - Thumbnail Thumbnail `json:"thumbnail"` -} - -type Thumbnail struct { - Src string `json:"src"` -} - -type Variant struct { - Id string `json:"id"` - OnSale bool `json:"onSale"` - Price Price `json:"price"` -} - -type Price struct { - Sales Sales `json:"sales"` - MarkdownDiscPercent int `json:"markdownDiscPercent"` -} - -type Sales struct { - Value float64 `json:"value"` - Currency string `json:"currency"` - Formatted string `json:"formatted"` - DecimalPrice string `json:"decimalPrice"` -} diff --git a/product/controller.go b/product/controller.go index 435119e..f378184 100644 --- a/product/controller.go +++ b/product/controller.go @@ -3,8 +3,8 @@ package product import ( "context" "fmt" - coach_client "gitea.timerzz.com/kedaya_haitao/coach-spider/pkg/coach-client" "gitea.timerzz.com/kedaya_haitao/common/model/product" + coach_client "gitea.timerzz.com/kedaya_haitao/common/pkg/coach-client" "github.com/golang/glog" "github.com/samber/lo" "gorm.io/gorm"