From 30d9fca1c50cbe3b0008848b0bca9d45883c703d Mon Sep 17 00:00:00 2001 From: Kuchen Date: Wed, 16 Oct 2024 11:08:34 +0200 Subject: [PATCH 1/4] + added support for custom judges check (with regex support) --- charm/hosts/loading_display.go | 2 +- common/judges.go | 55 ++++++++++++++++++++++++---------- common/settings.go | 46 +++++++++++++++------------- helper/checker_helper.go | 6 ++-- helper/thread_helper.go | 2 +- settings.json | 8 +++-- 6 files changed, 75 insertions(+), 44 deletions(-) diff --git a/charm/hosts/loading_display.go b/charm/hosts/loading_display.go index 5f4578e..94c9b45 100644 --- a/charm/hosts/loading_display.go +++ b/charm/hosts/loading_display.go @@ -107,7 +107,7 @@ func (m model) View() string { if len(data) == 0 { for _, val := range common.GetConfig().Judges { - data[val] = "?" + data[val.Url] = "?" } } diff --git a/common/judges.go b/common/judges.go index b47505a..e5a0b39 100644 --- a/common/judges.go +++ b/common/judges.go @@ -5,6 +5,7 @@ import ( "net" "net/http" "net/url" + "regexp" "sort" "strings" "sync" @@ -15,17 +16,20 @@ import ( type JudgesTimes struct { Judge string Ip string + Regex string ResponseTime time.Duration } type HostTimes []JudgesTimes var ( - UserIP string - FastestJudge string - FastestJudgeName *url.URL - FastestJudges map[string]string - FastestJudgesName map[string]*url.URL + UserIP string + FastestJudge string + FastestJudgeName *url.URL + FastestJudgeRegex string + FastestJudges map[string]string + FastestJudgesName map[string]*url.URL + FastestJudgesRegex map[string]string standardHeader = []string{"HTTP_HOST", "REQUEST_METHOD", "REMOTE_ADDR", "REMOTE_PORT"} ) @@ -52,6 +56,7 @@ var ( func CheckDomains() HostTimes { FastestJudges = make(map[string]string) FastestJudgesName = make(map[string]*url.URL) + FastestJudgesRegex = make(map[string]string) configHosts := GetConfig().Judges maxThreads := GetConfig().JudgesThreads @@ -77,6 +82,7 @@ func CheckDomains() HostTimes { sort.Sort(CurrentCheckedHosts) FastestJudge = CurrentCheckedHosts[0].Ip + FastestJudgeRegex = CurrentCheckedHosts[0].Regex u, err := url.Parse(CurrentCheckedHosts[0].Judge) if err == nil { @@ -91,6 +97,7 @@ func CheckDomains() HostTimes { if !ok { FastestJudges[protocol] = host.Ip + FastestJudgesRegex[protocol] = host.Regex } _, ok = FastestJudgesName[protocol] @@ -107,14 +114,15 @@ func CheckDomains() HostTimes { return unsortedHosts } -func checkTimeAsync(host string) { +func checkTimeAsync(host configJudge) { defer wg.Done() defer atomic.AddInt32(¤tThreads, -1) - ip, responseTime := checkTime(host) + ip, responseTime := checkTime(host.Url, host.Regex) hostTime := JudgesTimes{ - Judge: host, + Judge: host.Url, Ip: ip, + Regex: host.Regex, ResponseTime: responseTime, } @@ -124,7 +132,7 @@ func checkTimeAsync(host string) { } // Main function to check the time -func checkTime(host string) (string, time.Duration) { +func checkTime(host string, regex string) (string, time.Duration) { // Parse the URL to extract the hostname parsedURL, err := url.Parse(host) if err != nil { @@ -165,7 +173,7 @@ func checkTime(host string) (string, time.Duration) { return ip, time.Hour * 999 } - if !CheckForValidResponse(string(resBody)) { + if !CheckForValidResponse(string(resBody), regex) { return ip, time.Hour * 99 } @@ -212,12 +220,29 @@ func GetFastestJudgeNameForProtocol(protocol string) *url.URL { return FastestJudgesName[protocol] } -func CheckForValidResponse(html string) bool { - for _, header := range standardHeader { - if !strings.Contains(html, header) { - return false +func GetFastestJudgeRegexForProtocol(protocol string) string { + if strings.HasPrefix(protocol, "socks") { + return FastestJudgeRegex + } + + return FastestJudgesRegex[protocol] +} + +func CheckForValidResponse(html string, regex string) bool { + if strings.EqualFold(regex, "default") { + for _, header := range standardHeader { + if !strings.Contains(html, header) { + return false + } } + + return true + } + + re, err := regexp.Compile(regex) + if err != nil { + return false } - return true + return re.MatchString(html) } diff --git a/common/settings.go b/common/settings.go index bb9c14b..00028fc 100644 --- a/common/settings.go +++ b/common/settings.go @@ -9,22 +9,22 @@ import ( ) type Config struct { - Threads int `json:"threads"` - Retries int `json:"retries"` - Timeout int `json:"timeout"` - PrivacyMode bool `json:"privacy_mode"` - CopyToClipboard bool `json:"copyToClipboard"` - AutoSelect autoSelect `json:"autoSelect"` - AutoOutput autoSave `json:"autoSave"` - TimeBetweenRefresh int `json:"timeBetweenRefresh"` - IpLookup string `json:"iplookup"` - JudgesThreads int `json:"judges_threads"` - JudgesTimeOut int `json:"judges_timeout"` - Judges []string `json:"judges"` - Blacklisted []string `json:"blacklisted"` - Bancheck string `json:"bancheck"` - Keywords []string `json:"keywords"` - Transport transport `json:"transport"` + Threads int `json:"threads"` + Retries int `json:"retries"` + Timeout int `json:"timeout"` + PrivacyMode bool `json:"privacy_mode"` + CopyToClipboard bool `json:"copyToClipboard"` + AutoSelect autoSelect `json:"autoSelect"` + AutoOutput autoSave `json:"autoSave"` + TimeBetweenRefresh int `json:"timeBetweenRefresh"` + IpLookup string `json:"iplookup"` + JudgesThreads int `json:"judges_threads"` + JudgesTimeOut int `json:"judges_timeout"` + Judges []configJudge `json:"judges"` + Blacklisted []string `json:"blacklisted"` + Bancheck string `json:"bancheck"` + Keywords []string `json:"keywords"` + Transport transport `json:"transport"` } type autoSelect struct { @@ -52,6 +52,11 @@ type transport struct { ExpectContinueTimeout int `json:"ExpectContinueTimeout"` } +type configJudge struct { + Url string `json:"url"` + Regex string `json:"regex"` +} + var config Config func ReadSettings() { @@ -78,11 +83,11 @@ func RemoveHttpsJudges() { } func removeJudge(str string) { - var httpsJudges []string + var httpsJudges []configJudge for _, i2 := range config.Judges { - if strings.HasPrefix(i2, str) { - httpsJudges = append(httpsJudges, i2) + if strings.HasPrefix(i2.Url, str) { + httpsJudges = append(httpsJudges, configJudge{Url: i2.Url, Regex: i2.Regex}) } } @@ -114,12 +119,11 @@ func DoBanCheck() bool { } func IsAllowedToCheck(typeNames []string) bool { - for _, name := range typeNames { hasBeenFound := false for _, judge := range config.Judges { - if strings.HasPrefix(judge, name) { + if strings.HasPrefix(judge.Url, name) { hasBeenFound = true break } diff --git a/helper/checker_helper.go b/helper/checker_helper.go index 091e792..c624f6c 100644 --- a/helper/checker_helper.go +++ b/helper/checker_helper.go @@ -35,11 +35,11 @@ func GetProxyLevel(html string) int { } func Request(proxy *Proxy) (string, int, error) { - return RequestCustom(proxy, common.GetFastestJudgeForProtocol(proxy.Protocol), common.GetFastestJudgeNameForProtocol(proxy.Protocol), false) + return RequestCustom(proxy, common.GetFastestJudgeForProtocol(proxy.Protocol), common.GetFastestJudgeNameForProtocol(proxy.Protocol), common.GetFastestJudgeRegexForProtocol(proxy.Protocol), false) } // RequestCustom makes a request to the provided siteUrl with the provided proxy -func RequestCustom(proxyToCheck *Proxy, targetIp string, siteName *url.URL, isBanCheck bool) (string, int, error) { +func RequestCustom(proxyToCheck *Proxy, targetIp string, siteName *url.URL, regex string, isBanCheck bool) (string, int, error) { proxyURL, err := url.Parse(strings.Replace(proxyToCheck.Protocol, "https", "http", 1) + "://" + proxyToCheck.Full) if err != nil { return "Error parsing proxyToCheck URL", -1, err @@ -102,7 +102,7 @@ func RequestCustom(proxyToCheck *Proxy, targetIp string, siteName *url.URL, isBa } html := string(resBody) - if !isBanCheck && !common.CheckForValidResponse(html) { + if !isBanCheck && !common.CheckForValidResponse(html, regex) { return "Invalid response", -1, nil } diff --git a/helper/thread_helper.go b/helper/thread_helper.go index 9820d94..380257d 100644 --- a/helper/thread_helper.go +++ b/helper/thread_helper.go @@ -129,7 +129,7 @@ func check(proxy *Proxy) { if responded && common.DoBanCheck() && err == nil { for i := 0; i < retries; i++ { - body, status, err := RequestCustom(proxy, common.GetConfig().Bancheck, u, true) + body, status, err := RequestCustom(proxy, common.GetConfig().Bancheck, u, "", true) IncrementCheckCount() if err != nil { diff --git a/settings.json b/settings.json index 41c5ba4..2c544c1 100644 --- a/settings.json +++ b/settings.json @@ -1,5 +1,5 @@ { - "threads": 250, + "threads": 2000, "retries": 2, "timeout": 7500, @@ -29,8 +29,10 @@ "judges_threads": 3, "judges_timeout": 5000, "judges": [ - "http://azenv.net", - "https://pool.proxyspace.pro/judge.php" + {"url": "http://azenv.net", "regex": "default"}, + {"url": "https://pool.proxyspace.pro/judge.php", "regex": "default"}, + {"url": "https://ident.me/", "regex": "\\b(?:(?:25[0-5]|2[0-4]\\d|1\\d{2}|\\d{1,2})\\.){3}(?:25[0-5]|2[0-4]\\d|1\\d{2}|\\d{1,2})\\b"}, + {"url": "https://pool.proxyspace.pro/", "regex": "\\b(?:(?:25[0-5]|2[0-4]\\d|1\\d{2}|\\d{1,2})\\.){3}(?:25[0-5]|2[0-4]\\d|1\\d{2}|\\d{1,2})\\b"} ], "blacklisted": [ From 641fde5059920d67bdc8418e0a7a4419a338df6e Mon Sep 17 00:00:00 2001 From: Kuchen Date: Wed, 23 Oct 2024 08:51:37 +0200 Subject: [PATCH 2/4] + program doesn't crash anymore when you have too many threads --- main.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/main.go b/main.go index a53a62d..e9fa1e8 100644 --- a/main.go +++ b/main.go @@ -5,12 +5,15 @@ import ( "KC-Checker/common" "KC-Checker/helper" "github.com/jwalton/go-supportscolor" + "runtime/debug" ) func main() { //Lets the terminal on Windows 10 support true color supportscolor.Stdout() + debug.SetMaxThreads(999999999) + common.ReadSettings() common.GetLocalIP() From b9ab57731f8405f9d04b2e3dc2ed3674b59bbf53 Mon Sep 17 00:00:00 2001 From: Kuchen Date: Sun, 27 Oct 2024 13:12:10 +0100 Subject: [PATCH 3/4] + updated judge validity and settings --- common/judges.go | 6 +++++- settings.json | 7 ++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/common/judges.go b/common/judges.go index e5a0b39..ab343c9 100644 --- a/common/judges.go +++ b/common/judges.go @@ -31,7 +31,7 @@ var ( FastestJudgesName map[string]*url.URL FastestJudgesRegex map[string]string - standardHeader = []string{"HTTP_HOST", "REQUEST_METHOD", "REMOTE_ADDR", "REMOTE_PORT"} + standardHeader = []string{"USER-AGENT", "HOST", "ACCEPT", "ACCEPT-ENCODING", "ACCEPT-ENCODING"} ) func (ht HostTimes) Len() int { @@ -230,8 +230,12 @@ func GetFastestJudgeRegexForProtocol(protocol string) string { func CheckForValidResponse(html string, regex string) bool { if strings.EqualFold(regex, "default") { + html = strings.ReplaceAll(html, "_", "-") + html = strings.ToUpper(html) + for _, header := range standardHeader { if !strings.Contains(html, header) { + return false } } diff --git a/settings.json b/settings.json index 2c544c1..d4356a2 100644 --- a/settings.json +++ b/settings.json @@ -1,5 +1,5 @@ { - "threads": 2000, + "threads": 250, "retries": 2, "timeout": 7500, @@ -30,9 +30,10 @@ "judges_timeout": 5000, "judges": [ {"url": "http://azenv.net", "regex": "default"}, + {"url": "http://httpbin.org/headers", "regex": "default"}, {"url": "https://pool.proxyspace.pro/judge.php", "regex": "default"}, - {"url": "https://ident.me/", "regex": "\\b(?:(?:25[0-5]|2[0-4]\\d|1\\d{2}|\\d{1,2})\\.){3}(?:25[0-5]|2[0-4]\\d|1\\d{2}|\\d{1,2})\\b"}, - {"url": "https://pool.proxyspace.pro/", "regex": "\\b(?:(?:25[0-5]|2[0-4]\\d|1\\d{2}|\\d{1,2})\\.){3}(?:25[0-5]|2[0-4]\\d|1\\d{2}|\\d{1,2})\\b"} + {"url": "https://httpbingo.org/headers", "regex": "default"}, + {"url": "https://postman-echo.com/headers", "regex": "default"} ], "blacklisted": [ From 80e46a8bfa619a6f8bf59bc8286b0986cc706454 Mon Sep 17 00:00:00 2001 From: Kuchen Date: Sun, 27 Oct 2024 14:10:27 +0100 Subject: [PATCH 4/4] + updated readme --- README.MD | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.MD b/README.MD index 74a80f0..58d9989 100644 --- a/README.MD +++ b/README.MD @@ -181,7 +181,9 @@ Now you have an executable. You can run it like described in **Getting Started** Timeout duration of judges in ms 12. **judges**:
- These websites are the sites the proxies will connect to. These have to return the headers of the request so the checker can determine what anonymity level the proxy is. + **url**: This is the url the proxy will connect to while checking
+ **regex**: The proxy will only be valid if the regex is found. When "default" is used it will check the headers. Headers are needed to determine the anonymity level of the proxies.
+ If there are multiple judges the checker will use the fastest one 13. **blacklisted**:
Websites that contain blacklisted ips. These ips won't be checked