Replacement of inets with ibrowse. Fixes COUCHDB-179 and enhances rep…

…lication.

Thanks Jason Davies and Adam Kocoloski for the fix, Maximillian Dornseif for reporting.


git-svn-id: http://svn.apache.org/repos/asf/couchdb/trunk@739047 13f79535-47bb-0310-9956-ffa450edef68

CHANGES

@@ -28,6 +28,8 @@ Database Core:
    dramatically. The fix keeps only one document in the write queue at a time.
  * Fix for databases sometimes incorrectly reporting that they contain 0
    documents after compaction.
+ * CouchDB now uses ibrowse instead of inets for its internal HTTP client
+   implementation. This means better replication stability.
 
 HTTP Interface:
 

Makefile.am

@@ -10,7 +10,7 @@
 ## License for the specific language governing permissions and limitations
 ## under the License.
 
-SUBDIRS = bin etc src/couchdb src/mochiweb share test var utils
+SUBDIRS = bin etc src/couchdb src/ibrowse src/mochiweb share test var utils
 
 localdoc_DATA = AUTHORS.gz BUGS.gz CHANGES.gz NEWS.gz README.gz THANKS.gz
 

NOTICE

@@ -21,3 +21,9 @@ This product also includes the following third-party components:
  * MochiWeb (http://code.google.com/p/mochiweb/)
 
    Copyright 2007, Mochi Media Coporation
+
+ * ibrowse
+ (http://jungerl.cvs.sourceforge.net/viewvc/jungerl/jungerl/lib/ibrowse/)
+
+   Copyright 2008, Chandrashekhar Mullaparthi
+   This ASF redistribution is consistent with the terms of the BSD License.

THANKS

@@ -12,7 +12,9 @@ Some of these people are:
  * Yoan Blanc <[email protected]>
  * Paul Carey <[email protected]>
  * Benoit Chesneau <[email protected]>
+ * Jason Davies <[email protected]>
  * Paul Joseph Davis <[email protected]>
+ * Maximillian Dornseif <[email protected]>
  * Michael Gottesman <[email protected]>
  * Michael Hendricks <[email protected]>
  * Till Klampaeckel <[email protected]>

bin/Makefile.am

@@ -28,8 +28,9 @@ couchdb: couchdb.tpl
 	    -e "s|%ICU_CONFIG%|$(ICU_CONFIG)|g" \
 	    -e "s|%bindir%|@bindir@|g" \
 	    -e "s|%localerlanglibdir%|@localerlanglibdir@|g" \
-	    -e "s|%mochiwebebindir%|couch-@version@/ebin|g" \
-	    -e "s|%couchdbebindir%|mochiweb-r82/ebin|g" \
+	    -e "s|%couchdbebindir%|couch-@version@/ebin|g" \
+	    -e "s|%mochiwebebindir%|mochiweb-r82/ebin|g" \
+	    -e "s|%ibrowseebindir%|ibrowse-1.4.1/ebin|g" \
 	    -e "s|%defaultini%|default.ini|g" \
 	    -e "s|%localini%|local.ini|g" \
 	    -e "s|%localconfdir%|@localconfdir@|g" \

bin/couchdb.tpl.in

@@ -183,11 +183,12 @@ start_couchdb () {
         %ERL% $interactive_option -smp auto -sasl errlog_type error +K true \
         -pa %localerlanglibdir%/%couchdbebindir% \
             %localerlanglibdir%/%mochiwebebindir% \
-        -eval \"application:load(inets)\" \
+            %localerlanglibdir%/%ibrowseebindir% \
+        -eval \"application:load(ibrowse)\" \
         -eval \"application:load(crypto)\" \
         -eval \"application:load(couch)\" \
         -eval \"crypto:start()\" \
-        -eval \"inets:start()\" \
+        -eval \"ibrowse:start()\" \
         -eval \"couch_server:start([$start_arguments]), receive done -> done end.\" "
     if test "$BACKGROUND_BOOLEAN" = "true" \
         -a "$RECURSED_BOOLEAN" = "false"; then

configure.ac

@@ -267,6 +267,7 @@ AC_CONFIG_FILES([etc/Makefile])
 AC_CONFIG_FILES([share/Makefile])
 AC_CONFIG_FILES([src/couchdb/couch.app.tpl])
 AC_CONFIG_FILES([src/couchdb/Makefile])
+AC_CONFIG_FILES([src/ibrowse/Makefile])
 AC_CONFIG_FILES([src/mochiweb/Makefile])
 AC_CONFIG_FILES([test/Makefile])
 AC_CONFIG_FILES([utils/Makefile])

share/www/script/couch_tests.js

@@ -2134,6 +2134,19 @@ var tests = {
             T(docA._rev == docB._rev);
           };
         },
+
+        design_docs_test: new function() {
+          // make sure design docs replicate properly
+          this.init = function(dbA, dbB) {
+            dbA.save({ _id:"_design/test" });
+          };
+
+          this.afterAB1 = function() {
+            var docA = dbA.open("_design/test");
+            var docB = dbB.open("_design/test");
+            T(docA._rev == docB._rev);
+          };
+        },
 
         attachments_test: new function () {
           // Test attachments

src/couchdb/couch.app.tpl.in

@@ -24,4 +24,4 @@
                            couch_view,
                            couch_query_servers,
                            couch_db_update_notifier_sup]},
-              {applications,[kernel,stdlib,crypto,inets,mochiweb]}]}.
+              {applications,[kernel,stdlib,crypto,ibrowse,mochiweb]}]}.

src/couchdb/couch_httpd.erl

@@ -15,7 +15,7 @@
 
 -export([start_link/0, stop/0, handle_request/3]).
 
--export([header_value/2,header_value/3,qs_value/2,qs_value/3,qs/1,path/1]).
+-export([header_value/2,header_value/3,qs_value/2,qs_value/3,qs/1,path/1,absolute_uri/2]).
 -export([verify_is_server_admin/1,unquote/1,quote/1,recv/2]).
 -export([parse_form/1,json_body/1,body/1,doc_etag/1, make_etag/1, etag_respond/3]).
 -export([primary_header_value/2,partition/1,serve_file/3]).
@@ -242,6 +242,15 @@ qs(#httpd{mochi_req=MochiReq}) ->
 path(#httpd{mochi_req=MochiReq}) ->
     MochiReq:get(path).
 
+absolute_uri(#httpd{mochi_req=MochiReq}, Path) ->
+    Host = case MochiReq:get_header_value("Host") of
+        undefined ->
+            {ok, {Address, Port}} = inet:sockname(MochiReq:get(socket)),
+            inet_parse:ntoa(Address) ++ ":" ++ integer_to_list(Port);
+        Value -> Value
+    end,
+    "http://" ++ Host ++ Path.
+
 unquote(UrlEncodedString) ->
     mochiweb_util:unquote(UrlEncodedString).
 

src/couchdb/couch_httpd_db.erl

@@ -261,7 +261,8 @@ db_req(#httpd{method='GET',mochi_req=MochiReq, path_parts=[DbName,<<"_design/",_
     PathFront = "/" ++ couch_httpd:quote(binary_to_list(DbName)) ++ "/",
     RawSplit = regexp:split(MochiReq:get(raw_path),"_design%2F"),
     {ok, [PathFront|PathTail]} = RawSplit,
-    RedirectTo = PathFront ++ "_design/" ++ mochiweb_util:join(PathTail, "%2F"),
+    RedirectTo = couch_httpd:absolute_uri(Req, PathFront ++ "_design/" ++ 
+        mochiweb_util:join(PathTail, "%2F")),
     couch_httpd:send_response(Req, 301, [{"Location", RedirectTo}], <<>>);
 
 db_req(#httpd{path_parts=[_DbName,<<"_design">>,Name]}=Req, Db) ->

src/couchdb/couch_httpd_misc_handlers.erl

@@ -50,7 +50,8 @@ handle_utils_dir_req(#httpd{method='GET'}=Req, DocumentRoot) ->
         couch_httpd:serve_file(Req, RelativePath, DocumentRoot);
     {_ActionKey, "", _RelativePath} ->
         % GET /_utils
-        couch_httpd:send_response(Req, 301, [{"Location", "/_utils/"}], <<>>)
+        Headers = [{"Location", couch_httpd:absolute_uri(Req, "/_utils/")}],
+        couch_httpd:send_response(Req, 301, Headers, <<>>)
     end;
 handle_utils_dir_req(Req, _) ->
     send_method_not_allowed(Req, "GET,HEAD").

src/couchdb/couch_rep.erl

@@ -157,30 +157,54 @@ replicate2(Source, DbSrc, Target, DbTgt, Options) ->
     end.
 
 pull_rep(DbTarget, DbSource, SourceSeqNum) ->
-    http:set_options([{max_pipeline_length, 101}, {pipeline_timeout, 5000}]),
     {ok, {NewSeq, Stats}} = 
         enum_docs_since(DbSource, DbTarget, SourceSeqNum, {SourceSeqNum, []}),
-    http:set_options([{max_pipeline_length, 2}, {pipeline_timeout, 0}]),
     {NewSeq, Stats}.
 
 do_http_request(Url, Action, Headers) ->
     do_http_request(Url, Action, Headers, []).
 
 do_http_request(Url, Action, Headers, JsonBody) ->
-    ?LOG_DEBUG("couch_rep HTTP client request:", []),
-    ?LOG_DEBUG("\tAction: ~p", [Action]),
-    ?LOG_DEBUG("\tUrl: ~p", [Url]),
-    Request =
+    do_http_request(Url, Action, Headers, JsonBody, 10).
+
+do_http_request(Url, Action, _Headers, _JsonBody, 0) ->
+    ?LOG_ERROR("couch_rep HTTP ~p request failed after 10 retries: ~p", 
+        [Action, Url]);
+do_http_request(Url, Action, Headers, JsonBody, Retries) ->
+    ?LOG_DEBUG("couch_rep HTTP ~p request: ~p", [Action, Url]),
+    Body =
     case JsonBody of
     [] ->
-        {Url, Headers};
+        <<>>;
     _ ->
-        {Url, Headers, "application/json; charset=utf-8", iolist_to_binary(?JSON_ENCODE(JsonBody))}
+        iolist_to_binary(?JSON_ENCODE(JsonBody))
     end,
-    {ok, {{_, ResponseCode,_},_Headers, ResponseBody}} = http:request(Action, Request, [], []),
-    if
-    ResponseCode >= 200, ResponseCode < 500 ->
-        ?JSON_DECODE(ResponseBody)
+    Options = [
+        {content_type, "application/json; charset=utf-8"},
+        {max_pipeline_size, 101},
+        {transfer_encoding, {chunked, 65535}}
+    ],
+    case ibrowse:send_req(Url, Headers, Action, Body, Options) of
+    {ok, Status, ResponseHeaders, ResponseBody} ->
+        ResponseCode = list_to_integer(Status),
+        if
+        ResponseCode >= 200, ResponseCode < 300 ->
+            ?JSON_DECODE(ResponseBody);
+        ResponseCode >= 300, ResponseCode < 400 ->
+            RedirectUrl = mochiweb_headers:get_value("Location", 
+                mochiweb_headers:make(ResponseHeaders)),
+            do_http_request(RedirectUrl, Action, Headers, JsonBody, Retries-1);
+        ResponseCode >= 400, ResponseCode < 500 -> 
+            ?JSON_DECODE(ResponseBody);        
+        ResponseCode == 500 ->
+            ?LOG_INFO("retrying couch_rep HTTP ~p request due to 500 error: ~p",
+                [Action, Url]),
+            do_http_request(Url, Action, Headers, JsonBody, Retries - 1)
+        end;
+    {error, Reason} ->
+        ?LOG_INFO("retrying couch_rep HTTP ~p request due to {error, ~p}: ~p", 
+            [Action, Reason, Url]),
+        do_http_request(Url, Action, Headers, JsonBody, Retries - 1)
     end.
 
 save_docs_buffer(DbTarget, DocsBuffer, []) ->
@@ -223,20 +247,17 @@ wait_result({Pid,Ref}) ->
     {'DOWN', Ref, _, _, Reason} -> exit(Reason)
 end.
 
-enum_docs_parallel(DbS, DbT, DocInfoList) ->
-    UpdateSeqs = [D#doc_info.update_seq || D <- DocInfoList],
+enum_docs_parallel(DbS, DbT, InfoList) ->
+    UpdateSeqs = [Seq || {_, Seq, _, _} <- InfoList],
     SaveDocsPid = spawn_link(fun() -> save_docs_buffer(DbT,[],UpdateSeqs) end),
 
-    Stats = pmap(fun(SrcDocInfo) ->
-        #doc_info{id=Id,
-            rev=Rev,
-            conflict_revs=Conflicts,
-            deleted_conflict_revs=DelConflicts,
-            update_seq=Seq} = SrcDocInfo,
-        SrcRevs = [Rev | Conflicts] ++ DelConflicts,
-
-        case get_missing_revs(DbT, [{Id, SrcRevs}]) of
-        {ok, [{Id, MissingRevs}]} ->
+    Stats = pmap(fun({Id, Seq, SrcRevs, MissingRevs}) ->
+        case MissingRevs of
+        [] ->
+            SaveDocsPid ! {self(), skip, Seq},
+            receive got_it -> ok end,
+            [{missing_checked, length(SrcRevs)}];
+        _ ->
             {ok, DocResults} = open_doc_revs(DbS, Id, MissingRevs, [latest]),
 
             % only save successful reads
@@ -247,13 +268,9 @@ enum_docs_parallel(DbS, DbT, DocInfoList) ->
             receive got_it -> ok end,
             [{missing_checked, length(SrcRevs)},
              {missing_found, length(MissingRevs)},
-             {docs_read, length(Docs)}];
-        {ok, []} ->
-            SaveDocsPid ! {self(), skip, Seq},
-            receive got_it -> ok end,
-            [{missing_checked, length(SrcRevs)}]
-        end    
-    end, DocInfoList),
+             {docs_read, length(Docs)}]
+        end
+    end, InfoList),
 
     SaveDocsPid ! {self(), shutdown},
 
@@ -345,7 +362,22 @@ enum_docs_since(DbSource, DbTarget, StartSeq, InAcc) ->
     [] ->
         {ok, InAcc};
     _ ->
-        Stats = enum_docs_parallel(DbSource, DbTarget, DocInfoList),
+        UpdateSeqs = [D#doc_info.update_seq || D <- DocInfoList],
+        SrcRevsList = lists:map(fun(SrcDocInfo) ->
+            #doc_info{id=Id,
+                rev=Rev,
+                conflict_revs=Conflicts,
+                deleted_conflict_revs=DelConflicts
+            } = SrcDocInfo,
+            SrcRevs = [Rev | Conflicts] ++ DelConflicts,
+            {Id, SrcRevs}
+        end, DocInfoList),        
+        {ok, MissingRevsList} = get_missing_revs(DbTarget, SrcRevsList),
+        InfoList = lists:map(fun({{Id, SrcRevs}, Seq}) ->
+            MissingRevs = proplists:get_value(Id, MissingRevsList, []),
+            {Id, Seq, SrcRevs, MissingRevs}
+        end, lists:zip(SrcRevsList, UpdateSeqs)),
+        Stats = enum_docs_parallel(DbSource, DbTarget, InfoList),
         OldStats = element(2, InAcc),
         TotalStats = [
             {<<"missing_checked">>, 

src/couchdb/couch_server_sup.erl

@@ -102,7 +102,7 @@ start_server(IniFiles) ->
 
 
     % ensure these applications are running
-    application:start(inets),
+    application:start(ibrowse),
     application:start(crypto),
 
     {ok, Pid} = supervisor:start_link(

src/ibrowse/Makefile.am

@@ -0,0 +1,47 @@
+## Licensed under the Apache License, Version 2.0 (the "License"); you may not
+## use this file except in compliance with the License.  You may obtain a copy
+## of the License at
+##
+##   http://www.apache.org/licenses/LICENSE-2.0
+##
+## Unless required by applicable law or agreed to in writing, software
+## distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+## WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+## License for the specific language governing permissions and limitations under
+## the License.
+
+ibrowseebindir = $(localerlanglibdir)/ibrowse-1.4.1/ebin
+
+ibrowse_file_collection = \
+    ibrowse.erl \
+    ibrowse_app.erl \
+    ibrowse_http_client.erl \
+    ibrowse_lb.erl \
+    ibrowse_lib.erl \
+    ibrowse_sup.erl \
+    ibrowse_test.erl
+
+ibrowseebin_static_file = ibrowse.app
+
+ibrowseebin_make_generated_file_list = \
+    ibrowse.beam \
+    ibrowse_app.beam \
+    ibrowse_http_client.beam \
+    ibrowse_lb.beam \
+    ibrowse_lib.beam \
+    ibrowse_sup.beam \
+    ibrowse_test.beam
+
+ibrowseebin_DATA = \
+    $(ibrowseebin_static_file) \
+    $(ibrowseebin_make_generated_file_list)
+
+EXTRA_DIST =  \
+    $(ibrowse_file_collection) \
+    $(ibrowseebin_static_file)
+
+CLEANFILES = \
+    $(ibrowseebin_make_generated_file_list)
+
+%.beam: %.erl
+	$(ERLC) $<

src/ibrowse/ibrowse.app

@@ -0,0 +1,13 @@
+{application, ibrowse,
+        [{description, "HTTP client application"},
+         {vsn, "1.4.1"},
+         {modules, [ ibrowse, 
+		     ibrowse_http_client, 
+		     ibrowse_app, 
+		     ibrowse_sup, 
+		     ibrowse_lib,
+		     ibrowse_lb ]},
+         {registered, []},
+         {applications, [kernel,stdlib,sasl]},
+	 {env, []},
+	 {mod, {ibrowse_app, []}}]}.