proxy-detector/config/source.go

package config

import (
	"os"
	"time"

	"gopkg.in/yaml.v3"
)

type Source struct {
	Type    string     `json:"type" yaml:"type"`
	Options CrawOption `json:"options" yaml:"options"`
}

type CrawOption struct {
	Url        string `json:"url" yaml:"url"`
	DateFormat bool   `json:"date-format" yaml:"date-format"`
}

func (c *CrawOption) Urls() []string {
	var urls = make([]string, 0)
	url := c.Url
	if c.DateFormat {
		now := time.Now()
		urls = append(urls, now.Format(url))
		now = now.AddDate(0, 0, -1)
		urls = append(urls, now.Format(url))
	} else {
		urls = append(urls, url)
	}
	return urls
}

func LoadSource(path string) ([]Source, error) {
	var sources []Source
	f, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	defer f.Close()
	err = yaml.NewDecoder(f).Decode(&sources)
	return sources, err
}