Merge branch 'feature/parent-http-proxy' into develop

CHANGELOG

@@ -1,5 +1,6 @@
-0.5 (2013-02-01)
+0.5 (not released)
 
+    * Support parent HTTP proxy (such as goagent)
     * Work more automatically: because of this, updateBlocked, updateDirect, 
     autoRetry options and chou file are removed
     * Record direct/blocked visit count to make blocked/direct site handling

config.go

@@ -21,7 +21,8 @@ const (
 type Config struct {
 	RcFile        string // config file
 	ListenAddr    []string
-	SocksAddr     string
+	SocksParent   string
+	HttpParent    string
 	Core          int
 	SshServer     string
 	DetectSSLErr  bool
@@ -75,9 +76,10 @@ func parseCmdLineConfig() *Config {
 	flag.StringVar(&c.RcFile, "rc", path.Join(dsFile.dir, rcFname), "configuration file")
 	// Specifying listen default value to StringVar would override config file options
 	flag.StringVar(&listenAddr, "listen", "", "proxy server listen address, default to "+defaultListenAddr)
-	flag.StringVar(&c.SocksAddr, "socks", "", "socks proxy address")
+	flag.StringVar(&c.SocksParent, "socksParent", "", "parent socks proxy address")
+	flag.StringVar(&c.HttpParent, "httpParent", "", "parent http proxy address")
 	flag.IntVar(&c.Core, "core", 2, "number of cores to use")
-	flag.StringVar(&c.SshServer, "sshServer", "", "remote server which will ssh to and provide sock server")
+	flag.StringVar(&c.SshServer, "sshServer", "", "remote server which will ssh to and provide socks server")
 	flag.StringVar(&c.LogFile, "logFile", "", "write output to file")
 	flag.StringVar(&c.ShadowSocks, "shadowSocks", "", "shadowsocks server address")
 	flag.StringVar(&c.ShadowPasswd, "shadowPasswd", "", "shadowsocks password")
@@ -156,7 +158,7 @@ func (p configParser) ParseListen(val string) {
 
 func isServerAddrValid(val string) bool {
 	if val == "" {
-		return true
+		return false
 	}
 	_, port := splitHostPort(val)
 	if port == "" {
@@ -165,12 +167,32 @@ func isServerAddrValid(val string) bool {
 	return true
 }
 
+var hasSocksOrShadowSocksProxy bool
+var hasHttpParentProxy bool
+
 func (p configParser) ParseSocks(val string) {
+	fmt.Println("socks option is going to be renamed to socksParent in the future, please change it")
+	p.ParseSocksParent(val)
+}
+
+func (p configParser) ParseSocksParent(val string) {
+	if !isServerAddrValid(val) {
+		fmt.Println("parent socks server must have port specified")
+		os.Exit(1)
+	}
+	config.SocksParent = val
+	hasSocksOrShadowSocksProxy = true
+	parentProxyCreator = append(parentProxyCreator, createctSocksConnection)
+}
+
+func (p configParser) ParseHttpParent(val string) {
 	if !isServerAddrValid(val) {
-		fmt.Println("socks server must have port specified")
+		fmt.Println("parent http server must have port specified")
 		os.Exit(1)
 	}
-	config.SocksAddr = val
+	config.HttpParent = val
+	hasHttpParentProxy = true
+	parentProxyCreator = append(parentProxyCreator, createHttpProxyConnection)
 }
 
 func (p configParser) ParseCore(val string) {
@@ -214,6 +236,8 @@ func (p configParser) ParseShadowSocks(val string) {
 		os.Exit(1)
 	}
 	config.ShadowSocks = val
+	hasSocksOrShadowSocksProxy = true
+	parentProxyCreator = append(parentProxyCreator, createShadowSocksConnection)
 }
 
 func (p configParser) ParseShadowPasswd(val string) {

doc/sample-config/rc

@@ -11,19 +11,21 @@ logFile =
 # 总使用二级代理设置为 true 后，所有网站都通过二级代理访问
 #alwaysProxy = false
 
+# 若同时指定多个二级代理，COW 将按照配置文件中出现的顺序尝试连接二级代理
+
 #############################
 # SOCKS5 代理
 #############################
 
 # SOCKS5 代理地址
-socks = 127.0.0.1:1080
+socksParent = 127.0.0.1:1080
 
 # 你可以注释掉下面的选项，自己启动 socks 代理
 # 或者让 COW 执行 ssh 命令创建本地 socks 代理，并在 ssh 断开后重连
 # 注意这一功能需要配置好 ssh public key authentication
 #
 # 若指定该选项，COW 将执行以下命令：
-#     ssh -n -N -D <socks 选项中的端口> -p <sshServer 选项中的端口> <sshServer>
+#     ssh -n -N -D <socksParent 选项中的端口> -p <sshServer 选项中的端口> <sshServer>
 # sshServer 端口默认为 22。如果要指定其他 ssh 选项，请修改 ~/.ssh/config
 #sshServer = user@server[:port]
 
@@ -32,7 +34,6 @@ socks = 127.0.0.1:1080
 #############################
 
 # 若你有自己的 VPS，推荐使用 shadowsocks。服务器端部署完成后本机无需 socks 代理
-# 若同时指定 shadowsocks 和 SOCKS5 代理，优先使用 shadowsocks
 
 # shadowsocks 服务器地址，一定要指定端口
 #shadowSocks = 1.1.1.1:8838

http.go

@@ -31,10 +31,12 @@ const (
 )
 
 type Request struct {
-	Method  string
-	URL     *URL
-	contBuf *bytes.Buffer // will be non nil when retrying request
-	raw     bytes.Buffer
+	Method      string
+	URL         *URL
+	contBuf     *bytes.Buffer // will be non nil when retrying request
+	raw         bytes.Buffer
+	origReqLine []byte // original request line from client, used for http parent proxy
+	headerStart int    // start of header in raw
 	Header
 	isConnect bool
 	state     rqState
@@ -93,6 +95,7 @@ func (rp *Response) String() string {
 type URL struct {
 	HostPort string // must contain port
 	Host     string // no port
+	Port     string
 	Domain   string
 	Path     string
 	Scheme   string
@@ -113,7 +116,6 @@ func (url *URL) HostIsIP() bool {
 // For port, return empty string if no port specified.
 func splitHostPort(s string) (host, port string) {
 	// Common case should has no port, check the last char first
-	// Update: as port is always added, this is no longer common case in COW
 	if !IsDigit(s[len(s)-1]) {
 		return s, ""
 	}
@@ -155,26 +157,29 @@ func ParseRequestURI(rawurl string) (*URL, error) {
 	}
 
 	var hostport, host, port, path string
-	f = strings.SplitN(rest, "/", 2)
-	hostport = f[0]
-	if len(f) == 1 || f[1] == "" {
-		path = "/"
+	id := strings.Index(rest, "/")
+	if id == -1 {
+		hostport = rest
 	} else {
-		path = "/" + f[1]
+		hostport = rest[:id]
+		path = rest[id:]
 	}
+
 	// Must add port in host so it can be used as key to find the correct
 	// server connection.
 	// e.g. google.com:80 and google.com:443 should use different connections.
 	host, port = splitHostPort(hostport)
 	if port == "" {
 		if len(scheme) == 4 {
 			hostport = net.JoinHostPort(host, "80")
+			port = "80"
 		} else {
 			hostport = net.JoinHostPort(host, "443")
+			port = "443"
 		}
 	}
 
-	return &URL{hostport, host, host2Domain(host), path, scheme}, nil
+	return &URL{hostport, host, port, host2Domain(host), path, scheme}, nil
 }
 
 // headers of interest to a proxy
@@ -343,12 +348,20 @@ func parseRequest(c *clientConn) (r *Request, err error) {
 		return
 	}
 	if r.Method == "CONNECT" {
-		// Consume remaining header and just return. Headers are not used for
-		// CONNECT method.
 		r.isConnect = true
+	} else if hasSocksOrShadowSocksProxy {
+		// Generate normal HTTP request line
+		r.raw.WriteString(r.Method + " ")
+		r.raw.WriteString(r.URL.Path)
+		r.raw.WriteString(" HTTP/1.1\r\n")
 	}
 
-	r.genRequestLine()
+	// for http parent proxy, always need the initial request line
+	if hasHttpParentProxy {
+		r.origReqLine = make([]byte, len(s))
+		copy(r.origReqLine, s)
+		r.headerStart = r.raw.Len()
+	}
 
 	// Read request header
 	if err = r.parseHeader(reader, &r.raw, r.URL); err != nil {
@@ -363,11 +376,6 @@ func parseRequest(c *clientConn) (r *Request, err error) {
 	return
 }
 
-func (r *Request) genRequestLine() {
-	r.raw.WriteString(r.Method + " " + r.URL.Path)
-	r.raw.WriteString(" HTTP/1.1\r\n")
-}
-
 func (r *Request) responseNotSent() bool {
 	return r.state <= rsSent
 }

http_test.go

@@ -31,20 +31,20 @@ func TestParseRequestURI(t *testing.T) {
 		url    *URL
 	}{
 		// I'm really tired of typing google.com ...
-		{"http://www.g.com", &URL{"www.g.com:80", "www.g.com", "g.com", "/", "http"}},
-		{"http://plus.g.com/", &URL{"plus.g.com:80", "plus.g.com", "g.com", "/", "http"}},
-		{"https://g.com:80", &URL{"g.com:80", "g.com", "g.com", "/", "http"}},
-		{"http://mail.g.com:80/", &URL{"mail.g.com:80", "mail.g.com", "g.com", "/", "http"}},
-		{"http://g.com:80/ncr", &URL{"g.com:80", "g.com", "g.com", "/ncr", "http"}},
-		{"https://g.com/ncr/tree", &URL{"g.com:443", "g.com", "g.com", "/ncr/tree", "http"}},
-		{"www.g.com.hk:80/", &URL{"www.g.com.hk:80", "www.g.com.hk", "g.com.hk", "/", "http"}},
-		{"g.com.jp:80", &URL{"g.com.jp:80", "g.com.jp", "g.com.jp", "/", "http"}},
-		{"g.com", &URL{"g.com:80", "g.com", "g.com", "/", "http"}},
-		{"g.com:80/ncr", &URL{"g.com:80", "g.com", "g.com", "/ncr", "http"}},
-		{"g.com/ncr/tree", &URL{"g.com:80", "g.com", "g.com", "/ncr/tree", "http"}},
-		{"simplehost", &URL{"simplehost:80", "simplehost", "", "/", "http"}},
-		{"simplehost:8080", &URL{"simplehost:8080", "simplehost", "", "/", "http"}},
-		{"192.168.1.1:8080", &URL{"192.168.1.1:8080", "192.168.1.1", "", "/", "http"}},
+		{"http://www.g.com", &URL{"www.g.com:80", "www.g.com", "80", "g.com", "", "http"}},
+		{"http://plus.g.com/", &URL{"plus.g.com:80", "plus.g.com", "80", "g.com", "/", "http"}},
+		{"https://g.com:80", &URL{"g.com:80", "g.com", "80", "g.com", "", "http"}},
+		{"http://mail.g.com:80/", &URL{"mail.g.com:80", "mail.g.com", "80", "g.com", "/", "http"}},
+		{"http://g.com:80/ncr", &URL{"g.com:80", "g.com", "80", "g.com", "/ncr", "http"}},
+		{"https://g.com/ncr/tree", &URL{"g.com:443", "g.com", "443", "g.com", "/ncr/tree", "http"}},
+		{"www.g.com.hk:80/", &URL{"www.g.com.hk:80", "www.g.com.hk", "80", "g.com.hk", "/", "http"}},
+		{"g.com.jp:80", &URL{"g.com.jp:80", "g.com.jp", "80", "g.com.jp", "", "http"}},
+		{"g.com", &URL{"g.com:80", "g.com", "80", "g.com", "", "http"}},
+		{"g.com:8000/ncr", &URL{"g.com:8000", "g.com", "8000", "g.com", "/ncr", "http"}},
+		{"g.com/ncr/tree", &URL{"g.com:80", "g.com", "80", "g.com", "/ncr/tree", "http"}},
+		{"simplehost", &URL{"simplehost:80", "simplehost", "80", "", "", "http"}},
+		{"simplehost:8080", &URL{"simplehost:8080", "simplehost", "8080", "", "", "http"}},
+		{"192.168.1.1:8080/", &URL{"192.168.1.1:8080", "192.168.1.1", "8080", "", "/", "http"}},
 	}
 	for _, td := range testData {
 		url, err := ParseRequestURI(td.rawurl)
@@ -66,6 +66,9 @@ func TestParseRequestURI(t *testing.T) {
 		if url.Host != td.url.Host {
 			t.Error(td.rawurl, "parsed host wrong:", td.url.Host, "got", url.Host)
 		}
+		if url.Port != td.url.Port {
+			t.Error(td.rawurl, "parsed port wrong:", td.url.Port, "got", url.Port)
+		}
 		if url.Domain != td.url.Domain {
 			t.Error(td.rawurl, "parsed domain wrong:", td.url.Domain, "got", url.Domain)
 		}
@@ -74,67 +77,3 @@ func TestParseRequestURI(t *testing.T) {
 		}
 	}
 }
-
-func TestURLToURI(t *testing.T) {
-	var testData = []struct {
-		url URL
-		uri string
-	}{
-		{URL{HostPort: "google.com", Path: "/ncr", Scheme: "http"}, "http://google.com/ncr"},
-		{URL{HostPort: "www.google.com", Path: "/ncr", Scheme: "https"}, "https://www.google.com/ncr"},
-	}
-	for _, td := range testData {
-		if td.url.toURI() != td.uri {
-			t.Error("URL", td.url.String(), "toURI got", td.url.toURI(), "should be", td.uri)
-		}
-	}
-}
-
-func TestParseKeyValueList(t *testing.T) {
-	var testData = []struct {
-		str string
-		kvm map[string]string
-	}{
-		{"\tk1=\"v1\", k2=\"v2\",  \tk3=\"v3\"", map[string]string{"k1": "v1", "k2": "v2", "k3": "v3"}},
-		{"k1=\"v1\"", map[string]string{"k1": "v1"}},
-		{"", map[string]string{}},
-	}
-	for _, td := range testData {
-		kvm := parseKeyValueList(td.str)
-		if len(kvm) != len(td.kvm) {
-			t.Error("key value list parse error, element count not equal:", td.str, td.kvm)
-		}
-		for k, v := range td.kvm {
-			if kvm[k] != v {
-				t.Error("key value list parse error:", td.str, "for element:", k, v, "got:", kvm[k])
-			}
-		}
-	}
-}
-
-func TestParseKeepAlive(t *testing.T) {
-	h := new(Header)
-	h.parseKeepAlive([]byte(" timeout=1"), nil)
-	if h.KeepAlive != time.Second {
-		t.Error("timeout value 1 error, got:", h.KeepAlive)
-	}
-	h.parseKeepAlive([]byte(" timeout=10"), nil)
-	if h.KeepAlive != time.Second*10 {
-		t.Error("timeout value 10 error, got:", h.KeepAlive)
-	}
-	h.parseKeepAlive([]byte(" timeout=20,max=5"), nil)
-	if h.KeepAlive != time.Second*20 {
-		t.Error("timeout value 20 error, got:", h.KeepAlive)
-	}
-	// should not crash on invalid data
-	h.KeepAlive = time.Duration(0)
-	h.parseKeepAlive([]byte(" timeout="), nil)
-	if h.KeepAlive != time.Duration(0) {
-		t.Error("should not get timeout for invalid data")
-	}
-	h.KeepAlive = time.Duration(0)
-	h.parseKeepAlive([]byte(" timeout=,max=5"), nil)
-	if h.KeepAlive != time.Duration(0) {
-		t.Error("should not get timeout")
-	}
-}

main.go

@@ -46,16 +46,15 @@ func main() {
 	initAuth()
 	initSocksServer()
 	initShadowSocks()
+	initSiteStat()
+	initPAC()
 
-	if !hasSocksServer && !hasShadowSocksServer {
-		info.Println("no socks/shadowsocks server, can't handle blocked sites")
+	if len(parentProxyCreator) == 0 {
+		info.Println("no parent proxy server, can't handle blocked sites")
 	} else {
 		hasParentProxy = true
 	}
 
-	initSiteStat()
-	initPAC()
-
 	/*
 		if *cpuprofile != "" {
 			f, err := os.Create(*cpuprofile)