kedaya_haitao/coach-spider
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix 修复没有去重导致数据库存储问题

Browse Source
This commit is contained in:
timerzz 2024-05-14 20:29:28 +08:00
parent 075a4812e4
commit bedf3229b4
2 changed files with 6 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
pkg/coach-client/client.go
View File

@ -6,9 +6,9 @@ import (
"gitea.timerzz.com/kedaya_haitao/common/pkg/proxy" "gitea.timerzz.com/kedaya_haitao/common/pkg/proxy"
restry_pool "gitea.timerzz.com/kedaya_haitao/common/pkg/restry-pool" restry_pool "gitea.timerzz.com/kedaya_haitao/common/pkg/restry-pool"
"github.com/go-resty/resty/v2" "github.com/go-resty/resty/v2"
"github.com/golang/glog"
"github.com/pkg/errors" "github.com/pkg/errors"
proxy2 "github.com/timerzz/proxypool/pkg/proxy" proxy2 "github.com/timerzz/proxypool/pkg/proxy"
"log/slog"
"time" "time"
) )
@ -45,7 +45,7 @@ func tryRequest(ctx context.Context, urlPath string, respData any, proxyGetter f
} }
_, err := callByProxy(ctx, p, urlPath, respData) _, err := callByProxy(ctx, p, urlPath, respData)
if err != nil { if err != nil {
glog.Error(err) slog.Debug(err.Error())
continue continue
} }
return nil return nil

4
product/controller.go
View File

@ -96,6 +96,10 @@ func (c *Controller) saveRespData(list []coach_client.Product) error {
}) })
} }
} }
// 去重
products = lo.UniqBy(products, func(p database.Product) string {
return p.Pid
})
return c.db.Clauses(clause.OnConflict{ return c.db.Clauses(clause.OnConflict{
Columns: []clause.Column{{Name: "pid"}}, Columns: []clause.Column{{Name: "pid"}},
DoUpdates: clause.AssignmentColumns([]string{"name", "color", "link", "orderable", "us_price", "cny_price", "cal_mark", "freight", "rate"}), DoUpdates: clause.AssignmentColumns([]string{"name", "color", "link", "orderable", "us_price", "cny_price", "cal_mark", "freight", "rate"}),