use logrus

.gitignore

@@ -5,6 +5,7 @@
 *.so
 *.dylib
 bin/*
+tmp/*
 
 # Test binary, build with `go test -c`
 *.test

Dockerfile

@@ -13,4 +13,4 @@ RUN apk add --no-cache ca-certificates tzdata
 WORKDIR /proxypool-src
 COPY ./assets /proxypool-src/assets
 COPY --from=builder /proxypool /proxypool-src/
-ENTRYPOINT ["/proxypool-src/proxypool"]
+ENTRYPOINT ["/proxypool-src/proxypool", "-d"]

README_NEW.md

@@ -6,9 +6,10 @@ Clash客户端支持：
 TODO
 - [ ] 考虑更换grpool。
   节点过多时会卡在节点检测上，并且本地无法复现。本以为是内存问题，但观察内存时发现mem和cpu的占用实际上都非常少。TCP连接关闭正常并非网络阻塞。
-- [ ] 储存日志。
 
 ## New
+2020-11-28
+- 日志分级，输出到文件
 
 2020-11-26
 - 修复了一个导致vmess h2解析错误的bug= =

api/router.go

@@ -2,8 +2,8 @@ package api
 
 import (
 	binhtml "github.com/Sansui233/proxypool/internal/bindata/html"
+	"github.com/Sansui233/proxypool/log"
 	"html/template"
-	"log"
 	"net/http"
 	"os"
 	"strconv"
@@ -233,16 +233,16 @@ func setupRouter() {
 func Run() {
 	setupRouter()
 	servePort := config.Config.Port
-	envp := os.Getenv("PORT") // envp for heroku. DO NOT SET ENV PORT IN PERSONAL SERVER UNLESS YOU KNOW WHAT YOU ARE DOING
+	envp := os.Getenv("PORT") // environment port for heroku app
 	if envp != "" {
 		servePort = envp
 	}
 	// Run on this server
 	err := router.Run(":" + servePort)
 	if err != nil {
-		log.Fatal("[router.go] Web server starting failed. Make sure your port [", servePort, "] has not been used.", err)
+		log.Errorln("router: Web server starting failed. Make sure your port %s has not been used. \n%s", servePort, err.Error())
 	} else {
-		log.Println("Proxypool is serving on port: ", servePort)
+		log.Infoln("Proxypool is serving on port: %d", servePort)
 	}
 }
 

go.mod

@@ -31,7 +31,9 @@ require (
 	github.com/patrickmn/go-cache v2.1.0+incompatible
 	github.com/robertkrimen/otto v0.0.0-20200922221731-ef014fd054ac
 	github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca // indirect
+	github.com/sirupsen/logrus v1.7.0
 	github.com/temoto/robotstxt v1.1.1 // indirect
+	golang.org/x/sys v0.0.0-20201126233918-771906719818 // indirect
 	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
 	google.golang.org/appengine v1.6.6 // indirect
 	google.golang.org/protobuf v1.25.0 // indirect

go.sum

@@ -430,6 +430,8 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f h1:+Nyd8tzPX9R7BWHguqsrbFdRx
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68 h1:nxC68pudNYkKU6jWhgrqdreuFiOQWj1Fs7T3VrH4Pjw=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201126233918-771906719818 h1:f1CIuDlJhwANEC2MM87MBEVMr3jl5bifgsfj90XAF9c=
+golang.org/x/sys v0.0.0-20201126233918-771906719818/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=

internal/app/getter.go

@@ -2,7 +2,7 @@ package app
 
 import (
 	"errors"
-	"fmt"
+	"github.com/Sansui233/proxypool/log"
 
 	"github.com/Sansui233/proxypool/internal/cache"
 
@@ -32,23 +32,23 @@ func initGetters(sourceFiles []string) {
 	for _, path := range sourceFiles {
 		data, err := config.ReadFile(path)
 		if err != nil {
-			fmt.Errorf("Init SourceFile Error: %s\n", err.Error())
+			log.Errorln("Init SourceFile Error: %s\n", err.Error())
 			continue
 		}
 		sourceList := make([]config.Source, 0)
 		err = yaml.Unmarshal(data, &sourceList)
 		if err != nil {
-			fmt.Errorf("Init SourceFile Error: %s\n", err.Error())
+			log.Errorln("Init SourceFile Error: %s\n", err.Error())
 			continue
 		}
 		for _, source := range sourceList {
 			g, err := getter.NewGetter(source.Type, source.Options)
 			if err == nil && g != nil {
 				Getters = append(Getters, g)
-				fmt.Println("init getter:", source.Type, source.Options)
+				log.Debugln("init getter: %s %v", source.Type, source.Options)
 			}
 		}
 	}
-	fmt.Println("Getter count:", len(Getters))
+	log.Infoln("Getter count: %d", len(Getters))
 	cache.GettersCount = len(Getters)
 }

internal/app/task.go

@@ -1,10 +1,9 @@
 package app
 
 import (
-	"fmt"
 	"github.com/Sansui233/proxypool/config"
+	"github.com/Sansui233/proxypool/log"
 	"github.com/Sansui233/proxypool/pkg/healthcheck"
-	"log"
 	"sync"
 	"time"
 
@@ -21,7 +20,6 @@ func CrawlGo() {
 	var pc = make(chan proxy.Proxy)
 	for _, g := range Getters {
 		wg.Add(1)
-		// 并发执行抓取node并存到pc
 		go g.Get2ChanWG(pc, wg)
 	}
 	proxies := cache.GetProxies("allproxies")
@@ -30,7 +28,7 @@ func CrawlGo() {
 	if proxies == nil && dbProxies != nil {
 		cache.SetProxies("proxies", dbProxies)
 		cache.LastCrawlTime = "抓取中，已载入上次数据库数据"
-		fmt.Println("Database: loaded")
+		log.Infoln("Database: loaded")
 	}
 	if dbProxies != nil {
 		proxies = dbProxies.UniqAppendProxyList(proxies)
@@ -50,57 +48,55 @@ func CrawlGo() {
 		}
 	}
 
-	// 节点衍生并去重 -> 去重在上面做了，衍生的必要性不太明白
-	//proxies = proxies.Deduplication().Derive()
-	log.Println("CrawlGo unique proxy count:", len(proxies))
+	proxies = proxies.Derive()
+	log.Infoln("CrawlGo unique proxy count: %d", len(proxies))
 
-	// 去除Clash(windows)不支持的节点
+	// Clean Clash unsupported proxy because health check depends on clash
 	proxies = provider.Clash{
 		provider.Base{
 			Proxies: &proxies,
 		},
 	}.CleanProxies()
-	log.Println("CrawlGo clash supported proxy count:", len(proxies))
+	log.Infoln("CrawlGo clash supported proxy count: %d", len(proxies))
 
 	cache.SetProxies("allproxies", proxies)
 	cache.AllProxiesCount = proxies.Len()
-	log.Println("AllProxiesCount:", cache.AllProxiesCount)
+	log.Infoln("AllProxiesCount: %d", cache.AllProxiesCount)
 	cache.SSProxiesCount = proxies.TypeLen("ss")
-	log.Println("SSProxiesCount:", cache.SSProxiesCount)
+	log.Infoln("SSProxiesCount: %d", cache.SSProxiesCount)
 	cache.SSRProxiesCount = proxies.TypeLen("ssr")
-	log.Println("SSRProxiesCount:", cache.SSRProxiesCount)
+	log.Infoln("SSRProxiesCount: %d", cache.SSRProxiesCount)
 	cache.VmessProxiesCount = proxies.TypeLen("vmess")
-	log.Println("VmessProxiesCount:", cache.VmessProxiesCount)
+	log.Infoln("VmessProxiesCount: %d", cache.VmessProxiesCount)
 	cache.TrojanProxiesCount = proxies.TypeLen("trojan")
-	log.Println("TrojanProxiesCount:", cache.TrojanProxiesCount)
+	log.Infoln("TrojanProxiesCount: %d", cache.TrojanProxiesCount)
 	cache.LastCrawlTime = time.Now().In(location).Format("2006-01-02 15:04:05")
 
 	// 节点可用性检测，使用batchsize不能降低内存占用，只是为了看性能
-	log.Println("Now proceed proxy health check...")
+	log.Infoln("Now proceed proxy health check...")
 	b := 1000
 	round := len(proxies) / b
 	okproxies := make(proxy.ProxyList, 0)
 	for i := 0; i < round; i++ {
 		okproxies = append(okproxies, healthcheck.CleanBadProxiesWithGrpool(proxies[i*b:(i+1)*b])...)
-		log.Println("Checking round:", i)
+		log.Infoln("\tChecking round: %d", i)
 	}
 	okproxies = append(okproxies, healthcheck.CleanBadProxiesWithGrpool(proxies[round*b:])...)
 	proxies = okproxies
 
-	log.Println("CrawlGo clash usable proxy count:", len(proxies))
+	log.Infoln("CrawlGo clash usable proxy count: %d", len(proxies))
 
 	// 重命名节点名称为类似US_01的格式，并按国家排序
 	proxies.NameSetCounrty().Sort().NameAddIndex()
-	//proxies.NameReIndex()
-	log.Println("Proxy rename DONE!")
+	log.Infoln("Proxy rename DONE!")
 
 	// 可用节点存储
 	cache.SetProxies("proxies", proxies)
 	cache.UsefullProxiesCount = proxies.Len()
 	database.SaveProxyList(proxies)
 	database.ClearOldItems()
 
-	log.Println("Usablility checking done. Open", config.Config.Domain+":"+config.Config.Port, "to check")
+	log.Infoln("Usablility checking done. Open %s to check", config.Config.Domain+":"+config.Config.Port)
 
 	// 测速
 	speedTestNew(proxies)
@@ -118,7 +114,6 @@ func CrawlGo() {
 
 // Speed test for new proxies
 func speedTestNew(proxies proxy.ProxyList) {
-	// speed check
 	if config.Config.SpeedTest {
 		cache.IsSpeedTest = "已开启"
 		if config.Config.Timeout > 0 {
@@ -132,7 +127,6 @@ func speedTestNew(proxies proxy.ProxyList) {
 
 // Speed test for all proxies in proxy.ProxyList
 func SpeedTest(proxies proxy.ProxyList) {
-	// speed check
 	if config.Config.SpeedTest {
 		cache.IsSpeedTest = "已开启"
 		if config.Config.Timeout > 0 {

internal/cron/cron.go

@@ -3,9 +3,9 @@ package cron
 import (
 	"github.com/Sansui233/proxypool/config"
 	"github.com/Sansui233/proxypool/internal/cache"
+	"github.com/Sansui233/proxypool/log"
 	"github.com/Sansui233/proxypool/pkg/healthcheck"
 	"github.com/Sansui233/proxypool/pkg/provider"
-	"log"
 	"runtime"
 
 	"github.com/Sansui233/proxypool/internal/app"
@@ -22,18 +22,18 @@ func Cron() {
 func crawlTask() {
 	err := app.InitConfigAndGetters("")
 	if err != nil {
-		log.Println("[cron.go] config parse error:", err)
+		log.Errorln("[cron.go] config parse error: %s", err)
 	}
 	app.CrawlGo()
 	app.Getters = nil
 	runtime.GC()
 }
 
 func speedTestTask() {
-	log.Println("Doing speed test task...")
+	log.Infoln("Doing speed test task...")
 	err := config.Parse("")
 	if err != nil {
-		log.Println("[cron.go] config parse error:", err)
+		log.Errorln("[cron.go] config parse error: %s", err)
 	}
 	pl := cache.GetProxies("proxies")
 
@@ -52,17 +52,17 @@ func speedTestTask() {
 }
 
 func frequentSpeedTestTask() {
-	log.Println("Doing speed test task for active proxies...")
+	log.Infoln("Doing speed test task for active proxies...")
 	err := config.Parse("")
 	if err != nil {
-		log.Println("[cron.go] config parse error:", err)
+		log.Errorln("[cron.go] config parse error: %s", err)
 	}
 	pl_all := cache.GetProxies("proxies")
 	pl := healthcheck.ProxyStats.ReqCountThan(config.Config.ActiveFrequency, pl_all, true)
 	if len(pl) > int(config.Config.ActiveMaxNumber) {
 		pl = healthcheck.ProxyStats.SortProxiesBySpeed(pl)[:config.Config.ActiveMaxNumber]
 	}
-	log.Println("Active proxies count:", len(pl))
+	log.Infoln("Active proxies count: %d", len(pl))
 
 	app.SpeedTest(pl)
 	cache.SetString("clashproxies", provider.Clash{

internal/database/db.go

@@ -1,7 +1,7 @@
 package database
 
 import (
-	"log"
+	"github.com/Sansui233/proxypool/log"
 	"os"
 
 	"github.com/Sansui233/proxypool/config"
@@ -26,11 +26,10 @@ func connect() (err error) {
 		Logger: logger.Default.LogMode(logger.Silent),
 	})
 	if err == nil {
-		log.Println("Database: successfully connected to: ", DB.Name())
+		log.Infoln("database: successfully connected to: %s", DB.Name())
 	} else {
 		DB = nil
-		log.Println("\n\t\t[db.go] Database connection Info: ", err,
-			"\n\t\t[db.go] Use cache to store proxies")
+		log.Warnln("database connection Infoln: %s \n\t\tUse cache to store proxies", err.Error())
 	}
 	return
 }

internal/database/proxy.go

@@ -1,10 +1,9 @@
 package database
 
 import (
-	"fmt"
+	"github.com/Sansui233/proxypool/log"
 	"github.com/Sansui233/proxypool/pkg/proxy"
 	"gorm.io/gorm"
-	"log"
 	"time"
 )
 
@@ -25,11 +24,11 @@ func InitTables() {
 			return
 		}
 	}
-	// Warning: 自动迁移仅仅会创建表，缺少列和索引，并且不会改变现有列的类型或删除未使用的列以保护数据。
+	// Warnln: 自动迁移仅仅会创建表，缺少列和索引，并且不会改变现有列的类型或删除未使用的列以保护数据。
 	// 如更改表的Column请于数据库中操作
 	err := DB.AutoMigrate(&Proxy{})
 	if err != nil {
-		log.Println("\n\t\t[db/proxy.go] Database migration failed")
+		log.Errorln("\n\t\t[db/proxy.go] database migration failed")
 		panic(err)
 	}
 }
@@ -42,7 +41,7 @@ func SaveProxyList(pl proxy.ProxyList) {
 	DB.Transaction(func(tx *gorm.DB) error {
 		// Set All Usable to false
 		if err := DB.Model(&Proxy{}).Where("useable = ?", true).Update("useable", "false").Error; err != nil {
-			log.Println("\n\t\t[db/proxy.go] Reset useable to false failed: ", err)
+			log.Warnln("database: Reset useable to false failed: %s", err.Error())
 		}
 		// Create or Update proxies
 		for i := 0; i < pl.Len(); i++ {
@@ -57,13 +56,13 @@ func SaveProxyList(pl proxy.ProxyList) {
 				if uperr := DB.Model(&Proxy{}).Where("identifier = ?", p.Identifier).Updates(&Proxy{
 					Base: proxy.Base{Useable: true, Name: p.Name},
 				}).Error; uperr != nil {
-					log.Println("\n\t\t[db/proxy.go] DB Update failed: ",
-						"\n\t\t[db/proxy.go] When Created item: ", err,
-						"\n\t\t[db/proxy.go] When Updated item: ", uperr)
+					log.Warnln("\n\t\tdatabase: Update failed:"+
+						"\n\t\tdatabase: When Created item: %s"+
+						"\n\t\tdatabase: When Updated item: %s", err.Error(), uperr.Error())
 				}
 			}
 		}
-		fmt.Println("Database: Updated")
+		log.Infoln("database: Updated")
 		return nil
 	})
 }
@@ -100,11 +99,11 @@ func ClearOldItems() {
 		var count int64
 		DB.Model(&Proxy{}).Where("updated_at < ? AND useable = ?", lastWeek, false).Count(&count)
 		if count == 0 {
-			fmt.Println("Database: Nothing old to sweep") // TODO always this line?
+			log.Infoln("database: Nothing old to sweep") // TODO always this line?
 		} else {
-			log.Println("\n\t\t[db/proxy.go] Delete old item failed: ", err)
+			log.Warnln("database: Delete old item failed: %s", err)
 		}
 	} else {
-		fmt.Println("Database: Swept old and unusable proxies")
+		log.Infoln("database: Swept old and unusable proxies")
 	}
 }