kedaya_haitao/cn-coach-spider
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat cn client支持tls
Some checks failed
Build image / build (push) Failing after 10m4s
Details

Browse Source
This commit is contained in:
timerzz 2024-06-14 17:45:51 +08:00
parent df74a824b2
commit b39910ca54
4 changed files with 9 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.gitea/workflows/build-push.yml
View File

@ -18,3 +18,7 @@ jobs:
HTTP_PROXY: http://192.168.31.55:10809 HTTP_PROXY: http://192.168.31.55:10809
HTTPS_PROXY: http://192.168.31.55:10809 HTTPS_PROXY: http://192.168.31.55:10809
- run: ko build --bare ./cmd - run: ko build --bare ./cmd
env:
KO_DOCKER_REPO: 192.168.31.55:5000/kedaya/cn-coach-spider
HTTP_PROXY: http://192.168.31.55:10809
HTTPS_PROXY: http://192.168.31.55:10809

2
go.mod
View File

@ -5,7 +5,7 @@ go 1.22.2
toolchain go1.22.3 toolchain go1.22.3
require ( require (
gitea.timerzz.com/kedaya_haitao/common v0.0.0-20240614082731-77e0c69168ae gitea.timerzz.com/kedaya_haitao/common v0.0.0-20240614094207-8ba5c95a9cab
github.com/gofiber/fiber/v3 v3.0.0-beta.2 github.com/gofiber/fiber/v3 v3.0.0-beta.2
github.com/golang/glog v1.2.1 github.com/golang/glog v1.2.1
github.com/samber/lo v1.39.0 github.com/samber/lo v1.39.0

4
go.sum
View File

@ -1,6 +1,6 @@
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
gitea.timerzz.com/kedaya_haitao/common v0.0.0-20240614082731-77e0c69168ae h1:/8+PiBgyPsOdhuXT7ZUv+OHrYXJwy/wDUwhui+yQnXE= gitea.timerzz.com/kedaya_haitao/common v0.0.0-20240614094207-8ba5c95a9cab h1:QaWiCysNQiIXVyy8tFe4WAY+2Mm7+izuAcBx2iJk3l0=
gitea.timerzz.com/kedaya_haitao/common v0.0.0-20240614082731-77e0c69168ae/go.mod h1:MAmc7ooakm0o+4YshgKb+uE3Mfhex10s2RcJMiIYick= gitea.timerzz.com/kedaya_haitao/common v0.0.0-20240614094207-8ba5c95a9cab/go.mod h1:MAmc7ooakm0o+4YshgKb+uE3Mfhex10s2RcJMiIYick=
github.com/3andne/restls-client-go v0.1.6 h1:tRx/YilqW7iHpgmEL4E1D8dAsuB0tFF3uvncS+B6I08= github.com/3andne/restls-client-go v0.1.6 h1:tRx/YilqW7iHpgmEL4E1D8dAsuB0tFF3uvncS+B6I08=
github.com/3andne/restls-client-go v0.1.6/go.mod h1:iEdTZNt9kzPIxjIGSMScUFSBrUH6bFRNg0BWlP4orEY= github.com/3andne/restls-client-go v0.1.6/go.mod h1:iEdTZNt9kzPIxjIGSMScUFSBrUH6bFRNg0BWlP4orEY=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=

2
spider/controller.go
View File

@ -61,7 +61,7 @@ func (c *Controller) Run(ctx context.Context) {
func (c *Controller) Crawl() error { func (c *Controller) Crawl() error {
glog.Info("开始抓取信息") glog.Info("开始抓取信息")
for page, totalPage := 0, 0; page <= totalPage; page++ { for page, totalPage := 1, 1; page <= totalPage; page++ {
resp, err := c.client.ListItems(c.ctx, page, 50) resp, err := c.client.ListItems(c.ctx, page, 50)
if err != nil { if err != nil {
return fmt.Errorf("访问coach第%d页失败: %w", page, err) return fmt.Errorf("访问coach第%d页失败: %w", page, err)