feat: support Alibaba Cloud mirror site. voidint#60

cli/install.go

@@ -12,6 +12,8 @@ import (
 	"github.com/dixonwille/wmenu/v5"
 	"github.com/mholt/archiver/v3"
 	"github.com/urfave/cli/v2"
+	"github.com/voidint/g/collector"
+	"github.com/voidint/g/collector/official"
 	"github.com/voidint/g/version"
 )
 
@@ -29,11 +31,11 @@ func install(ctx *cli.Context) (err error) {
 
 	var url string
 	if url = os.Getenv(mirrorEnv); url == "" {
-		url = version.DefaultURL
+		url = official.DefaultDownloadPageURL
 	}
 
 	// 查找版本
-	c, err := version.NewCollector(url)
+	c, err := collector.NewCollector(url)
 	if err != nil {
 		return cli.Exit(errstring(err), 1)
 	}

cli/ls_remote.go

@@ -7,6 +7,8 @@ import (
 	"github.com/Masterminds/semver"
 	"github.com/k0kubun/go-ansi"
 	"github.com/urfave/cli/v2"
+	"github.com/voidint/g/collector"
+	"github.com/voidint/g/collector/official"
 	"github.com/voidint/g/version"
 )
 
@@ -24,10 +26,10 @@ func listRemote(ctx *cli.Context) (err error) {
 
 	var url string
 	if url = os.Getenv(mirrorEnv); url == "" {
-		url = version.DefaultURL
+		url = official.DefaultDownloadPageURL
 	}
 
-	c, err := version.NewCollector(url)
+	c, err := collector.NewCollector(url)
 	if err != nil {
 		return cli.Exit(errstring(err), 1)
 	}

cli/update.go

@@ -12,6 +12,7 @@ import (
 	"github.com/voidint/g/build"
 	"github.com/voidint/g/github"
 	"github.com/voidint/g/pkg/checksum"
+	"github.com/voidint/g/pkg/errs"
 )
 
 func update(*cli.Context) (err error) {
@@ -65,7 +66,7 @@ func findChecksum(items []github.Asset) (algo checksum.Algorithm, expectedChecks
 		}
 	}
 	if checksumFileURL == "" {
-		return checksum.SHA256, "", checksum.ErrChecksumFileNotFound
+		return checksum.SHA256, "", errs.ErrChecksumFileNotFound
 	}
 
 	resp, err := http.Get(checksumFileURL)
@@ -85,5 +86,5 @@ func findChecksum(items []github.Asset) (algo checksum.Algorithm, expectedChecks
 	if err = scanner.Err(); err != nil {
 		return checksum.SHA256, "", err
 	}
-	return checksum.SHA256, "", checksum.ErrChecksumFileNotFound
+	return checksum.SHA256, "", errs.ErrChecksumFileNotFound
 }

collector/aliyun/aliyun_collector.go

@@ -0,0 +1,99 @@
+package aliyun
+
+import (
+	"net/http"
+	stdurl "net/url"
+	"strings"
+
+	"github.com/PuerkitoBio/goquery"
+	"github.com/voidint/g/pkg/errs"
+	"github.com/voidint/g/version"
+)
+
+// var _ collector.Collector = (*Collector)(nil)
+
+const (
+	// DownloadPageURL 阿里云镜像站点网址
+	DownloadPageURL = "https://mirrors.aliyun.com/golang/"
+)
+
+// Collector 阿里云镜像站点版本采集器
+type Collector struct {
+	url  string
+	pURL *stdurl.URL
+	doc  *goquery.Document
+}
+
+// NewCollector 返回采集器实例
+func NewCollector() (*Collector, error) {
+	pURL, err := stdurl.Parse(DownloadPageURL)
+	if err != nil {
+		return nil, err
+	}
+
+	c := Collector{
+		url:  DownloadPageURL,
+		pURL: pURL,
+	}
+	if err = c.loadDocument(); err != nil {
+		return nil, err
+	}
+	return &c, nil
+}
+
+func (c *Collector) loadDocument() (err error) {
+	resp, err := http.Get(c.url)
+	if err != nil {
+		return errs.NewURLUnreachableError(c.url, err)
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		return errs.NewURLUnreachableError(c.url, nil)
+	}
+	c.doc, err = goquery.NewDocumentFromReader(resp.Body)
+	return err
+}
+
+// StableVersions 返回所有稳定版本
+func (c *Collector) StableVersions() (items []*version.Version, err error) {
+	return make([]*version.Version, 0), nil
+}
+
+// UnstableVersions 返回所有非稳定版本
+func (c *Collector) UnstableVersions() (items []*version.Version, err error) {
+	return make([]*version.Version, 0), nil
+}
+
+// ArchivedVersions 返回已归档版本
+func (c *Collector) ArchivedVersions() (items []*version.Version, err error) {
+	return make([]*version.Version, 0), nil
+}
+
+// AllVersions 返回所有已知版本
+func (c *Collector) AllVersions() (vers []*version.Version, err error) {
+	items := c.findGoFileItems(c.doc.Find(".table"))
+	if len(items) == 0 {
+		return make([]*version.Version, 0, 0), nil
+	}
+	return convert2Versions(items), nil
+}
+
+func (c *Collector) findGoFileItems(table *goquery.Selection) (items []*goFileItem) {
+	trs := table.Find("tbody").Find("tr")
+	items = make([]*goFileItem, 0, trs.Length())
+
+	trs.Each(func(j int, tr *goquery.Selection) {
+		td := tr.Find("td")
+		href := td.Eq(0).Find("a").AttrOr("href", "")
+		if !strings.HasPrefix(href, "go") {
+			return
+		}
+
+		items = append(items, &goFileItem{
+			FileName: td.Eq(0).Find("a").Text(),
+			URL:      c.url + href,
+			Size:     td.Eq(1).Text(),
+		})
+	})
+	return items
+}

collector/aliyun/aliyun_collector_test.go

@@ -0,0 +1,36 @@
+package aliyun
+
+import (
+	"bytes"
+	"io/ioutil"
+	"testing"
+
+	"github.com/PuerkitoBio/goquery"
+	"github.com/stretchr/testify/assert"
+)
+
+func getCollector() (*Collector, error) {
+	b, err := ioutil.ReadFile("./testdata/golang_dl.html")
+	if err != nil {
+		return nil, err
+	}
+	doc, err := goquery.NewDocumentFromReader(bytes.NewBuffer(b))
+	if err != nil {
+		return nil, err
+	}
+	return &Collector{
+		url: DownloadPageURL,
+		doc: doc,
+	}, nil
+}
+
+func Test_findGoFileItems(t *testing.T) {
+	c, err := getCollector()
+	assert.Nil(t, err)
+	assert.NotNil(t, c)
+
+	t.Run("", func(t *testing.T) {
+		items := c.findGoFileItems(c.doc.Find(".table"))
+		assert.NotEmpty(t, items)
+	})
+}

collector/aliyun/go_file_item.go

@@ -0,0 +1,130 @@
+package aliyun
+
+import (
+	"fmt"
+	"strings"
+	"unicode"
+
+	"github.com/voidint/g/pkg/checksum"
+	"github.com/voidint/g/version"
+)
+
+type goFileItem struct {
+	FileName string
+	URL      string
+	Size     string
+}
+
+func (item goFileItem) getGoVersion() string {
+	arr := strings.Split(strings.TrimPrefix(item.FileName, "go"), ".")
+	if len(arr) < 3 || !unicode.IsNumber(rune(arr[0][0])) {
+		return ""
+	}
+	if unicode.IsNumber(rune(arr[2][0])) {
+		return fmt.Sprintf("%s.%s.%s", arr[0], arr[1], arr[2])
+	}
+
+	if unicode.IsNumber(rune(arr[1][0])) {
+		return fmt.Sprintf("%s.%s", arr[0], arr[1])
+	}
+	return arr[0]
+}
+
+func (item goFileItem) isSHA256File() bool {
+	return strings.HasSuffix(item.FileName, ".sha256")
+}
+
+func (item goFileItem) isPackageFile() bool {
+	return strings.HasSuffix(item.FileName, ".tar.gz") ||
+		strings.HasSuffix(item.FileName, ".pkg") ||
+		strings.HasSuffix(item.FileName, ".zip") ||
+		strings.HasSuffix(item.FileName, ".msi")
+}
+
+func (item goFileItem) getKind() string {
+	if strings.HasSuffix(item.FileName, ".src.tar.gz") {
+		return "Source"
+	}
+	if strings.HasSuffix(item.FileName, ".tar.gz") || strings.HasSuffix(item.FileName, ".zip") {
+		return "Archive"
+	}
+	if strings.HasSuffix(item.FileName, ".pkg") || strings.HasSuffix(item.FileName, ".msi") {
+		return "Installer"
+	}
+	return "Unknown"
+}
+
+var osMapping = map[string]string{
+	"darwin":  "macOS",
+	"linux":   "Linux",
+	"windows": "Windows",
+	"freebsd": "FreeBSD",
+}
+
+func (item goFileItem) getOS() string {
+	for k, v := range osMapping {
+		if strings.Contains(item.FileName, k) {
+			return v
+		}
+	}
+	return ""
+}
+
+var archMapping = map[string]string{
+	"amd64":   "x86-64",
+	"386":     "x86",
+	"arm64":   "ARM64",
+	"armv6l":  "ARMv6",
+	"ppc64le": "ppc64le",
+	"s390x":   "s390x",
+}
+
+func (item goFileItem) getArch() string {
+	for k, v := range archMapping {
+		if strings.Contains(item.FileName, k) {
+			return v
+		}
+	}
+	return ""
+}
+
+func convert2Versions(items []*goFileItem) (vers []*version.Version) {
+	pkgMap := make(map[string][]*version.Package, 20)
+
+	for _, pitem := range items {
+		ver := pitem.getGoVersion()
+		if _, ok := pkgMap[ver]; !ok {
+			pkgMap[ver] = make([]*version.Package, 0, 20)
+		}
+
+		if pitem.isPackageFile() {
+			pkgMap[ver] = append(pkgMap[ver], &version.Package{
+				FileName: pitem.FileName,
+				URL:      pitem.URL,
+				Kind:     pitem.getKind(),
+				OS:       pitem.getOS(),
+				Arch:     pitem.getArch(),
+				Size:     pitem.Size,
+			})
+		} else if pitem.isSHA256File() {
+			// 设置校验和及算法
+			for _, ppkg := range pkgMap[ver] {
+				if !strings.HasPrefix(pitem.FileName, ppkg.FileName) {
+					continue
+				}
+				ppkg.Algorithm = string(checksum.SHA256)
+				ppkg.ChecksumURL = pitem.URL
+			}
+		}
+	}
+
+	vers = make([]*version.Version, 0, len(pkgMap))
+	for k, v := range pkgMap {
+		vers = append(vers, &version.Version{
+			Name:     k,
+			Packages: v,
+		})
+	}
+
+	return vers
+}