feat: add Statistics API support

assets/css/main.css

@@ -5,6 +5,12 @@ body {
 	padding: 3.5rem 0;
 }
 
+@media screen and (max-width: 600px) {
+	body {
+		font-size: 0.875rem;
+	}
+}
+
 hr {
 	margin: 4rem 0;
 }

go.mod

@@ -10,7 +10,9 @@ require (
 	github.com/air-gases/redirector v0.15.2
 	github.com/aofei/air v0.15.2
 	github.com/fsnotify/fsnotify v1.4.9
-	github.com/goproxy/goproxy v0.0.0-20200317114900-cc9448f870be
+	github.com/goproxy/goproxy v0.0.0-20200326092349-8b22ef89a531
+	github.com/gorilla/websocket v1.4.2 // indirect
+	github.com/mitchellh/mapstructure v1.2.2 // indirect
 	github.com/pelletier/go-toml v1.6.0 // indirect
 	github.com/robfig/cron/v3 v3.0.1
 	github.com/rs/zerolog v1.18.0
@@ -22,5 +24,6 @@ require (
 	github.com/tdewolff/minify/v2 v2.7.3 // indirect
 	github.com/tidwall/gjson v1.6.0
 	github.com/tidwall/pretty v1.0.1 // indirect
-	github.com/yuin/goldmark v1.1.25
+	github.com/yuin/goldmark v1.1.26
+	golang.org/x/mod v0.2.0
 )

go.sum

@@ -77,11 +77,13 @@ github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
-github.com/goproxy/goproxy v0.0.0-20200317114900-cc9448f870be h1:PUysD7+2kogvHQgc1c/ddDm9L9IbXcyiOAeXreaUg04=
-github.com/goproxy/goproxy v0.0.0-20200317114900-cc9448f870be/go.mod h1:dBPGiwiZ1sLGodPVjRyGTomYyh+G6iph+8pWSiWQgfA=
+github.com/goproxy/goproxy v0.0.0-20200326092349-8b22ef89a531 h1:XD/TZ3Ju7pAk/3pTdrqGTqhjlCHWC27HxFHqk9v9X4w=
+github.com/goproxy/goproxy v0.0.0-20200326092349-8b22ef89a531/go.mod h1:rU10pOYYNEYeRcN+RB5nxge/WZHjGPqJaWZJsQ8kt60=
 github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
 github.com/gorilla/websocket v1.4.1 h1:q7AeDBpnBk8AogcD4DSag/Ukw/KV+YhzLj2bP5HvKCM=
 github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
+github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc=
+github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
 github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
@@ -112,6 +114,8 @@ github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/mitchellh/mapstructure v1.2.2 h1:dxe5oCinTXiTIcfgmZecdCzPmAJKd46KsCWc35r0TV4=
+github.com/mitchellh/mapstructure v1.2.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
 github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
@@ -197,8 +201,8 @@ github.com/vmihailenco/msgpack v4.0.4+incompatible h1:dSLoQfGFAo3F6OoNhwUmLwVgaU
 github.com/vmihailenco/msgpack v4.0.4+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
-github.com/yuin/goldmark v1.1.25 h1:isv+Q6HQAmmL2Ofcmg8QauBmDPlUUnSoNhEcC940Rds=
-github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.1.26 h1:81MfIApzizD8JqKIjnWy2Vxj7GgmQUrJlh/jYWH8yGk=
+github.com/yuin/goldmark v1.1.26/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q=
 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
@@ -210,8 +214,8 @@ golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8U
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073 h1:xMPOj6Pz6UipU1wXLkrtqpHbR0AVFnyPEQq/wRWz9lM=
 golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.0.0-20200311171314-f7b00557c8c4 h1:QmwruyY+bKbDDL0BaglrbZABEali68eoMFhTZpCjYVA=
-golang.org/x/crypto v0.0.0-20200311171314-f7b00557c8c4/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59 h1:3zb4D3T4G8jdExgVU/95+vQXfpEPiMdCaZgmGVxjNHM=
+golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/mod v0.2.0 h1:KU7oHjnv3XNWfa5COkzUifxZmxp1TyI7ImMXqFxLwvQ=
@@ -226,6 +230,8 @@ golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200301022130-244492dfa37a h1:GuSPYbZzB5/dcLNCwLQLsg3obCJtX9IJhpXkvY7kzk0=
 golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e h1:3G+cUijn7XD+S4eJFddp53Pv7+slrESplyjG25HgL+k=
+golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -242,8 +248,8 @@ golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527 h1:uYVVQ9WP/Ds2ROhcaGPeIdVq0RIXVLwsHlnvJ+cT1So=
 golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200317113312-5766fd39f98d h1:62ap6LNOjDU6uGmKXHJbSfciMoV+FeI1sRXx/pLDL44=
-golang.org/x/sys v0.0.0-20200317113312-5766fd39f98d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=

handler/goproxy.go

@@ -19,7 +19,6 @@ import (
 	"github.com/aofei/air"
 	"github.com/goproxy/goproxy"
 	"github.com/goproxy/goproxy.cn/base"
-	"github.com/goproxy/goproxy/cacher"
 )
 
 var (
@@ -70,12 +69,7 @@ func init() {
 	})
 
 	hhGoproxy.Cacher = &goproxyCacher{
-		Cacher: &cacher.Kodo{
-			Endpoint:   qiniuViper.GetString("kodo_endpoint"),
-			AccessKey:  qiniuAccessKey,
-			SecretKey:  qiniuSecretKey,
-			BucketName: qiniuViper.GetString("kodo_bucket_name"),
-		},
+		Cacher:         qiniuKodoCacher,
 		localCacheRoot: goproxyLocalCacheRoot,
 		settingContext: ctx,
 	}

handler/handler.go

@@ -1,21 +1,17 @@
 package handler
 
 import (
-	"bytes"
 	"context"
-	"crypto/hmac"
-	"crypto/sha1"
-	"encoding/base64"
-	"fmt"
-	"io"
+	"errors"
 	"io/ioutil"
 	"net/http"
 	"strconv"
-	"time"
+	"strings"
 
 	"github.com/air-gases/cacheman"
 	"github.com/aofei/air"
 	"github.com/goproxy/goproxy.cn/base"
+	"github.com/goproxy/goproxy/cacher"
 	"github.com/tidwall/gjson"
 )
 
@@ -29,6 +25,14 @@ var (
 	// qiniuSecretKey is the secret key for the Qiniu Cloud.
 	qiniuSecretKey = qiniuViper.GetString("secret_key")
 
+	// qiniuKodoCacher is the kodo cacher for the Qiniu Cloud.
+	qiniuKodoCacher = &cacher.Kodo{
+		Endpoint:   qiniuViper.GetString("kodo_endpoint"),
+		AccessKey:  qiniuAccessKey,
+		SecretKey:  qiniuSecretKey,
+		BucketName: qiniuViper.GetString("kodo_bucket_name"),
+	}
+
 	// getHeadMethods is an array contains the GET and the HEAD methods.
 	getHeadMethods = []string{http.MethodGet, http.MethodHead}
 
@@ -39,24 +43,24 @@ var (
 		SMaxAge: -1,
 	})
 
-	// cachedModuleVersionCount is the cached module version count.
-	cachedModuleVersionCount int64
+	// moduleVersionCount is the module version count.
+	moduleVersionCount int64
 )
 
 func init() {
-	updateCachedModuleVersionsCount()
-	if cachedModuleVersionCount == 0 {
+	updateModuleVersionsCount()
+	if moduleVersionCount == 0 {
 		base.Logger.Fatal().
-			Msg("failed to initialize cached module version count")
+			Msg("failed to initialize module version count")
 	}
 
 	if _, err := base.Cron.AddFunc(
-		"0 0 * * * *", // every 1 hour
-		updateCachedModuleVersionsCount,
+		"0 */10 * * * *", // every 10 minutes
+		updateModuleVersionsCount,
 	); err != nil {
 		base.Logger.Fatal().Err(err).
-			Msg("failed to add cached module version count " +
-				"update cron job")
+			Msg("failed to add module version count update cron " +
+				"job")
 	}
 
 	base.Air.FILE("/robots.txt", "robots.txt")
@@ -72,122 +76,63 @@ func init() {
 	base.Air.BATCH(getHeadMethods, "/", hIndexPage)
 }
 
+// NotFoundHandler handles not found.
+func NotFoundHandler(req *air.Request, res *air.Response) error {
+	res.Status = http.StatusNotFound
+	return errors.New(strings.ToLower(http.StatusText(res.Status)))
+}
+
+// MethodNotAllowedHandler handles method not allowed.
+func MethodNotAllowedHandler(req *air.Request, res *air.Response) error {
+	res.Status = http.StatusMethodNotAllowed
+	return errors.New(strings.ToLower(http.StatusText(res.Status)))
+}
+
 // Error handles errors.
 func Error(err error, req *air.Request, res *air.Response) {
 	if res.Written {
 		return
 	}
 
-	m := ""
 	if !req.Air.DebugMode && res.Status == http.StatusInternalServerError {
-		m = http.StatusText(res.Status)
+		res.WriteString(strings.ToLower(http.StatusText(res.Status)))
 	} else {
-		m = err.Error()
+		res.WriteString(err.Error())
 	}
-
-	res.WriteJSON(map[string]interface{}{
-		"Error": m,
-	})
 }
 
 // hIndexPage handles requests to get index page.
 func hIndexPage(req *air.Request, res *air.Response) error {
 	return res.Render(map[string]interface{}{
 		"IsIndexPage": true,
-		"CachedModuleVersionCount": thousandsCommaSeperated(
-			cachedModuleVersionCount,
+		"ModuleVersionCount": thousandsCommaSeperated(
+			moduleVersionCount,
 		),
 	}, req.LocalizedString("index.html"), "layouts/default.html")
 }
 
-// updateCachedModuleVersionsCount updates the `cachedModuleVersionCount`.
-func updateCachedModuleVersionsCount() {
+// updateModuleVersionsCount updates the `moduleVersionCount`.
+func updateModuleVersionsCount() {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 	defer base.Air.RemoveShutdownJob(base.Air.AddShutdownJob(cancel))
 
-	b, err := requestQiniuAPI(
-		ctx,
-		http.MethodGet,
-		fmt.Sprintf(
-			"https://api.qiniu.com"+
-				"/v6/count?bucket=%s&begin=%s&end=%s&g=day",
-			qiniuViper.GetString("kodo_bucket_name"),
-			time.Now().Add(-time.Hour).In(base.TZAsiaShanghai).
-				Format("20060102150405"),
-			time.Now().In(base.TZAsiaShanghai).
-				Format("20060102150405"),
-		),
-		"",
-		nil,
-	)
+	cache, err := qiniuKodoCacher.Cache(ctx, "stats/summary")
 	if err != nil {
 		base.Logger.Error().Err(err).
-			Msg("failed to update cached module version count")
-	}
-
-	count := gjson.GetBytes(b, "datas.0").Int()
-	if count > 0 {
-		cachedModuleVersionCount = count / 3
-	}
-}
-
-// requestQiniuAPI requests Qiniu API.
-func requestQiniuAPI(
-	ctx context.Context,
-	method string,
-	url string,
-	contentType string,
-	body io.Reader,
-) ([]byte, error) {
-	req, err := http.NewRequestWithContext(ctx, method, url, body)
-	if err != nil {
-		return nil, err
-	}
-
-	path := req.URL.Path
-	if req.URL.RawQuery != "" {
-		path = fmt.Sprint(path, "?", req.URL.RawQuery)
-	}
-
-	data := []byte(fmt.Sprint(path, "\n"))
-	if contentType == "application/x-www-form-urlencoded" && body != nil {
-		b, err := ioutil.ReadAll(body)
-		if err != nil {
-			return nil, err
-		}
-
-		data = append(data, b...)
-
-		req.Body = ioutil.NopCloser(bytes.NewReader(b))
-	}
-
-	h := hmac.New(sha1.New, []byte(qiniuSecretKey))
-	h.Write(data)
-	sign := base64.URLEncoding.EncodeToString(h.Sum(nil))
-	token := fmt.Sprint(qiniuAccessKey, ":", sign)
-
-	req.Header.Set("Authorization", fmt.Sprint("QBox ", token))
-	if contentType != "" {
-		req.Header.Set("Content-Type", contentType)
-	}
-
-	res, err := base.HTTPDo(nil, req)
-	if err != nil {
-		return nil, err
+			Msg("failed to update module version count")
+		return
 	}
-	defer res.Body.Close()
+	defer cache.Close()
 
-	b, err := ioutil.ReadAll(res.Body)
+	b, err := ioutil.ReadAll(cache)
 	if err != nil {
-		return nil, err
-	}
-
-	if res.StatusCode == http.StatusOK {
-		return b, nil
+		base.Logger.Error().Err(err).
+			Msg("failed to update module version count")
+		return
 	}
 
-	return nil, fmt.Errorf("GET %s: %s: %s", url, res.Status, b)
+	moduleVersionCount = gjson.GetBytes(b, "module_version_count").Int()
 }
 
 // thousandsCommaSeperated returns a thousands comma seperated string for the n.