This commit is contained in:
parent
3eeed3d11a
commit
3202eb827b
12
cmd/main.go
12
cmd/main.go
@ -11,9 +11,12 @@ import (
|
|||||||
"github.com/gofiber/fiber/v3"
|
"github.com/gofiber/fiber/v3"
|
||||||
"github.com/gofiber/fiber/v3/middleware/cors"
|
"github.com/gofiber/fiber/v3/middleware/cors"
|
||||||
"github.com/golang/glog"
|
"github.com/golang/glog"
|
||||||
|
"gorm.io/gorm/logger"
|
||||||
|
"log"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"os"
|
"os"
|
||||||
"os/signal"
|
"os/signal"
|
||||||
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
@ -28,6 +31,15 @@ func main() {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
glog.Fatalf("初始化数据库失败:%v", err)
|
glog.Fatalf("初始化数据库失败:%v", err)
|
||||||
}
|
}
|
||||||
|
db.Logger = logger.New(
|
||||||
|
log.New(os.Stdout, "\r\n", log.LstdFlags),
|
||||||
|
logger.Config{
|
||||||
|
SlowThreshold: time.Second,
|
||||||
|
LogLevel: logger.Warn,
|
||||||
|
IgnoreRecordNotFoundError: false,
|
||||||
|
Colorful: true,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
// 代理池
|
// 代理池
|
||||||
pool := proxy.NewProxyPool(cfg.Proxy.Subscribes)
|
pool := proxy.NewProxyPool(cfg.Proxy.Subscribes)
|
||||||
|
@ -40,8 +40,8 @@ type Variant struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type Price struct {
|
type Price struct {
|
||||||
Sales Sales `json:"sales"`
|
Sales Sales `json:"sales"`
|
||||||
MarkdownDiscPercent float32 `json:"markdownDiscPercent"`
|
MarkdownDiscPercent int `json:"markdownDiscPercent"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type Sales struct {
|
type Sales struct {
|
||||||
|
@ -5,6 +5,7 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
coach_client "gitea.timerzz.com/kedaya_haitao/coach-spider/pkg/coach-client"
|
coach_client "gitea.timerzz.com/kedaya_haitao/coach-spider/pkg/coach-client"
|
||||||
"gitea.timerzz.com/kedaya_haitao/common/model/product"
|
"gitea.timerzz.com/kedaya_haitao/common/model/product"
|
||||||
|
"github.com/golang/glog"
|
||||||
"github.com/samber/lo"
|
"github.com/samber/lo"
|
||||||
"gorm.io/gorm"
|
"gorm.io/gorm"
|
||||||
"gorm.io/gorm/clause"
|
"gorm.io/gorm/clause"
|
||||||
@ -72,6 +73,7 @@ func (c *Controller) Crawl() error {
|
|||||||
if err = c.saveRespData(resp.PageData.Products); err != nil {
|
if err = c.saveRespData(resp.PageData.Products); err != nil {
|
||||||
return fmt.Errorf("保存第%d页数据失败: %w", page, err)
|
return fmt.Errorf("保存第%d页数据失败: %w", page, err)
|
||||||
}
|
}
|
||||||
|
glog.Infof("第%d页数据保存成功", page)
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@ -90,13 +92,14 @@ func (c *Controller) saveRespData(list []coach_client.Product) error {
|
|||||||
c.Freight = saveFreight
|
c.Freight = saveFreight
|
||||||
}
|
}
|
||||||
products = append(products, productv1.Product{
|
products = append(products, productv1.Product{
|
||||||
|
UpdatedAt: time.Now(),
|
||||||
Name: resp.Name,
|
Name: resp.Name,
|
||||||
Pid: color.VgId,
|
Pid: color.VgId,
|
||||||
Color: color.Text,
|
Color: color.Text,
|
||||||
Link: fmt.Sprintf("%s/%s", "https://www.coachoutlet.com", color.Url),
|
Link: fmt.Sprintf("%s/%s", "https://www.coachoutlet.com", color.Url),
|
||||||
Image: color.Media.Thumbnail.Src,
|
Image: color.Media.Thumbnail.Src,
|
||||||
Orderable: color.Orderable,
|
Orderable: color.Orderable,
|
||||||
DiscPercent: int(price.Price.MarkdownDiscPercent),
|
DiscPercent: price.Price.MarkdownDiscPercent,
|
||||||
USPrice: price.Price.Sales.Value,
|
USPrice: price.Price.Sales.Value,
|
||||||
Freight: c.Freight,
|
Freight: c.Freight,
|
||||||
ExchangeRate: c.ExchangeRate,
|
ExchangeRate: c.ExchangeRate,
|
||||||
@ -109,6 +112,6 @@ func (c *Controller) saveRespData(list []coach_client.Product) error {
|
|||||||
})
|
})
|
||||||
return c.db.Clauses(clause.OnConflict{
|
return c.db.Clauses(clause.OnConflict{
|
||||||
Columns: []clause.Column{{Name: "pid"}},
|
Columns: []clause.Column{{Name: "pid"}},
|
||||||
DoUpdates: clause.AssignmentColumns([]string{"name", "color", "link", "orderable", "us_price", "cny_price", "cal_mark", "rate", "price_status", "disc_percent"}),
|
DoUpdates: clause.AssignmentColumns([]string{"name", "color", "link", "orderable", "us_price", "cny_price", "cal_mark", "rate", "price_status", "disc_percent", "updated_at"}),
|
||||||
}).Create(products).Error
|
}).Create(products).Error
|
||||||
}
|
}
|
||||||
|
Loading…
Reference in New Issue
Block a user