Skip to content

Commit

Permalink
add geo parse
Browse files Browse the repository at this point in the history
  • Loading branch information
zu1k committed Aug 16, 2020
1 parent e13892a commit 23a337e
Show file tree
Hide file tree
Showing 11 changed files with 101 additions and 10 deletions.
12 changes: 12 additions & 0 deletions app/global.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
package app

import "github.com/zu1k/proxypool/proxy"

var (
GeoIp proxy.GeoIP
ProjectName = "proxypool"
)

func init() {
GeoIp = proxy.NewGeoIP("assets/GeoLite2-City.mmdb")
}
10 changes: 6 additions & 4 deletions app/task.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,6 @@ import (
"strconv"
"sync"

"github.com/zu1k/proxypool/checker"

"github.com/zu1k/proxypool/config"
"gopkg.in/yaml.v2"

Expand Down Expand Up @@ -64,12 +62,16 @@ func CrawlGo() {
}
proxies = proxy.Deduplication(proxies)
log.Println("CrawlGo node count:", len(proxies))
proxies = checker.CleanProxies(provider.Clash{Proxies: proxies}.CleanProxies())
proxies = proxy.CleanProxies(provider.Clash{Proxies: proxies}.CleanProxies())
log.Println("CrawlGo clash useable node count:", len(proxies))

num := len(proxies)
for i := 0; i < num; i++ {
proxies[i].SetName("tgbot.co_" + strconv.Itoa(i+1))
country, err := GeoIp.Find(proxies[i].BaseInfo().Server)
if err != nil {
country = "Earth"
}
proxies[i].SetName(fmt.Sprintf("%s_%d_%s", ProjectName, i+1, country))
}
cache.SetProxies(proxies)
cache.SetString("clashproxies", provider.Clash{Proxies: proxies}.Provide())
Expand Down
Binary file added assets/GeoLite2-City.mmdb
Binary file not shown.
2 changes: 2 additions & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -22,12 +22,14 @@ require (
github.com/miekg/dns v1.1.31 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.1 // indirect
github.com/oschwald/geoip2-golang v1.4.0
github.com/oschwald/maxminddb-golang v1.7.0 // indirect
github.com/patrickmn/go-cache v2.1.0+incompatible
github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca // indirect
github.com/spf13/cobra v1.0.0 // indirect
github.com/spf13/viper v1.7.1 // indirect
github.com/temoto/robotstxt v1.1.1 // indirect
github.com/zu1k/nali v0.2.2 // indirect
golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de // indirect
golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc // indirect
golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208 // indirect
Expand Down
18 changes: 18 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -164,6 +164,8 @@ github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/J
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
github.com/ipipdotnet/ipdb-go v1.3.0 h1:FfkSkAI1do3bZ7F35ueGuF7Phur64jmikQ1C4IPl/gc=
github.com/ipipdotnet/ipdb-go v1.3.0/go.mod h1:yZ+8puwe3R37a/3qRftXo40nZVQbxYDLqls9o5foexs=
github.com/jasonlvhit/gocron v0.0.1 h1:qTt5qF3b3srDjeOIR4Le1LfeyvoYzJlYpqvG7tJX5YU=
github.com/jasonlvhit/gocron v0.0.1/go.mod h1:k9a3TV8VcU73XZxfVHCHWMWF9SOqgoku0/QlY2yvlA4=
github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
Expand Down Expand Up @@ -258,6 +260,11 @@ github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD
github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca h1:NugYot0LIVPxTvN8n+Kvkn6TrbMyxQiuvKdEwFdR9vI=
github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU=
github.com/saracen/go7z v0.0.0-20191010121135-9c09b6bd7fda h1:h+YpzUB/bGVJcLqW+d5GghcCmE/A25KbzjXvWJQi/+o=
github.com/saracen/go7z v0.0.0-20191010121135-9c09b6bd7fda/go.mod h1:MSotTrCv1PwoR8QgU1JurEx+lNNbtr25I+m0zbLyAGw=
github.com/saracen/go7z-fixtures v0.0.0-20190623165746-aa6b8fba1d2f/go.mod h1:6Ff0ADODZ6S3gYepgZ2w7OqFrTqtFcfwDUhmm8jsUhs=
github.com/saracen/solidblock v0.0.0-20190426153529-45df20abab6f h1:1cJITU3JUI8qNS5T0BlXwANsVdyoJQHQ4hvOxbunPCw=
github.com/saracen/solidblock v0.0.0-20190426153529-45df20abab6f/go.mod h1:LyBTue+RWeyIfN3ZJ4wVxvDuvlGJtDgCLgCb6HCPgps=
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
Expand All @@ -277,6 +284,8 @@ github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9
github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE=
github.com/spf13/viper v1.7.1 h1:pM5oEahlgWv/WnHXpgbKz7iLIxRf65tye2Ci+XFK5sk=
github.com/spf13/viper v1.7.1/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg=
Expand All @@ -298,8 +307,12 @@ github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo=
github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw=
github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs=
github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY=
github.com/ulikunitz/xz v0.5.7 h1:YvTNdFzX6+W5m9msiYg/zpkSURPPtOlzbqYjrFn7Yt4=
github.com/ulikunitz/xz v0.5.7/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
github.com/zu1k/nali v0.2.2 h1:LRo52LXqdhwbjCFOs5Bl9XGewvJnd3MC92m5hDPFHyc=
github.com/zu1k/nali v0.2.2/go.mod h1:5BILGeMPfJz+sv3Sov25g8kkNGgB/ouj5lYSD39SUuc=
go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
Expand Down Expand Up @@ -396,6 +409,7 @@ golang.org/x/sys v0.0.0-20191224085550-c709ea063b76/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884=
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200810151505-1b9f1253b3ed h1:WBkVNH1zd9jg/dK4HCM4lNANnmd12EHC9z+LmcCG4ns=
golang.org/x/sys v0.0.0-20200810151505-1b9f1253b3ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed h1:J22ig1FUekjjkmZUM7pTKixYm8DvrYsvrBZdunYeIuQ=
Expand All @@ -404,6 +418,8 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
Expand All @@ -424,6 +440,7 @@ golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtn
golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191216052735-49a3e744a425 h1:VvQyQJN0tSuecqgcIxMWnnfG5kSmgy9KZR9sW3W5QeA=
golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
Expand Down Expand Up @@ -492,5 +509,6 @@ honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWh
honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM=
honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
1 change: 1 addition & 0 deletions proxy/base.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ type Proxy interface {
Identifier() string
SetName(name string)
Type() string
BaseInfo() *Base
}

func Deduplication(src []Proxy) []Proxy {
Expand Down
11 changes: 5 additions & 6 deletions checker/check.go → proxy/check.go
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package checker
package proxy

import (
"context"
Expand All @@ -7,12 +7,11 @@ import (
"time"

"github.com/Dreamacro/clash/adapters/outbound"
"github.com/zu1k/proxypool/proxy"
)

const defaultURLTestTimeout = time.Second * 15

func Check(p proxy.Proxy) (delay uint16, err error) {
func Check(p Proxy) (delay uint16, err error) {
pmap := make(map[string]interface{})
err = json.Unmarshal([]byte(p.String()), &pmap)
if err != nil {
Expand All @@ -35,7 +34,7 @@ func Check(p proxy.Proxy) (delay uint16, err error) {
return delay, err
}

func CleanProxies(proxies []proxy.Proxy) (cproxies []proxy.Proxy) {
func CleanProxies(proxies []Proxy) (cproxies []Proxy) {
c := make(chan checkResult, 40)
for _, p := range proxies {
go checkProxyWithChan(p, c)
Expand All @@ -48,7 +47,7 @@ func CleanProxies(proxies []proxy.Proxy) (cproxies []proxy.Proxy) {
okMap[r.name] = struct{}{}
}
}
cproxies = make([]proxy.Proxy, 0)
cproxies = make([]Proxy, 0)
for _, p := range proxies {
if _, ok := okMap[p.Identifier()]; ok {
cproxies = append(cproxies, p)
Expand All @@ -62,7 +61,7 @@ type checkResult struct {
delay uint16
}

func checkProxyWithChan(p proxy.Proxy, c chan checkResult) {
func checkProxyWithChan(p Proxy, c chan checkResult) {
delay, err := Check(p)
if err != nil {
c <- checkResult{
Expand Down
45 changes: 45 additions & 0 deletions proxy/geoip.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
package proxy

import (
"log"
"net"
"os"

"github.com/oschwald/geoip2-golang"
)

// GeoIP2
type GeoIP struct {
db *geoip2.Reader
}

// new geoip from db file
func NewGeoIP(filePath string) (geoip GeoIP) {
// 判断文件是否存在
_, err := os.Stat(filePath)
if err != nil && os.IsNotExist(err) {
log.Println("文件不存在,请自行下载 Geoip2 City库,并保存在", filePath)
os.Exit(1)
} else {
db, err := geoip2.Open(filePath)
if err != nil {
log.Fatal(err)
}
geoip = GeoIP{db: db}
}
return
}

// find ip info
func (g GeoIP) Find(ipORdomain string) (country string, err error) {
ips, err := net.LookupIP(ipORdomain)
if err != nil {
return "", err
}
ipData := net.ParseIP(ips[0].String())
record, err := g.db.City(ipData)
if err != nil {
return "", err
}
return record.Country.IsoCode, nil
}
4 changes: 4 additions & 0 deletions proxy/shadowsocks.go
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,10 @@ func (ss *Shadowsocks) Type() string {
return "ss"
}

func (ss *Shadowsocks) BaseInfo() *Base {
return &ss.Base
}

func ParseSSLink(link string) (*Shadowsocks, error) {
if !strings.HasPrefix(link, "ss://") {
return nil, ErrorNotSSRLink
Expand Down
4 changes: 4 additions & 0 deletions proxy/shadowsocksr.go
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,10 @@ func (ssr *ShadowsocksR) Type() string {
return "ssr"
}

func (ssr *ShadowsocksR) BaseInfo() *Base {
return &ssr.Base
}

func ParseSSRLink(link string) (*ShadowsocksR, error) {
if !strings.HasPrefix(link, "ssr") {
return nil, ErrorNotSSRLink
Expand Down
4 changes: 4 additions & 0 deletions proxy/vmess.go
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,10 @@ func (v *Vmess) Type() string {
return "vmess"
}

func (v *Vmess) BaseInfo() *Base {
return &v.Base
}

type vmessLinkJson struct {
Add string `json:"add"`
V string `json:"v"`
Expand Down

0 comments on commit 23a337e

Please sign in to comment.