fix sphinx builder query range bug; improve sphinx builder db query performance, to avoid `getmore' command and the 4M useless response

src/db_store_mongo.erl

@@ -71,7 +71,7 @@ search(Conn, Key) when is_list(Key) ->
 search_announce_top(Conn, Count) ->
 	Sel = {'$query', {}, '$orderby', {announce, -1}},
 	List = mongo_do_slave(Conn, fun() ->
-		% mongodb-erlang does not provide cursor.limit()/sort() functions, wired
+		% mongodb-erlang does not provide cursor.limit()/sort() functions, weird
 		% but it work here
 		Cursor = mongo:find(?COLLNAME, Sel, [], 0, Count), 
 		mongo_cursor:rest(Cursor)
@@ -343,3 +343,27 @@ test_compile() ->
     io:format("sphins disabled~n", []).
 -endif.
 
+% about mongodb `batchsize' and `getmore':http://blog.nosqlfan.com/html/3996.html
+% looks like the `batchsize' argument will take effect on `getmore' command
+% 1. load `Batch' docs first, then take them all
+% Cursor = mongo:find(?COLLNAME, {}, [], 0, Batch),
+% mongo_cursor:take(Cursor, Batch) 
+% 2. default load 101 docs first
+% Cursor = mongo:find(?COLLNAME, {}, [], 0, 0),
+% mongo_cursor:take(Cursor, Batch) 
+% 3. default load 101 docs first, then `getmore' to load 4M data
+% Cursor = mongo:find(?COLLNAME, {}, [], 0, 0),
+% mongo_cursor:take(Cursor, 102)
+% 4. load a batch of docs, and mongodb close the connection
+% Cursor = mongo:find(?COLLNAME, {}, [], 0, -Cnt),
+% mongo_cursor:rest(Cursor)
+test_batch(Cnt) ->
+	Conn = mongo_pool:get(db_pool),
+	mongo:do(safe, master, Conn, ?DBNAME, fun() ->
+		%Cursor = mongo:find(?COLLNAME, {}, [], 0, Cnt),
+		Cursor = mongo:find(?COLLNAME, {}, [], 0, -Cnt),
+		%mongo_cursor:take(Cursor, Cnt)
+		%mongo_cursor:take(Cursor, Cnt)
+		mongo_cursor:rest(Cursor)
+	end).
+

src/sphinx_builder/sphinx_torrent.erl

@@ -112,7 +112,7 @@ forward_date(Date) ->
 
 % will cause lots of queries
 do_load_torrents(Date, Size) ->
-	Q = {created_at, {'$gt', Date, '$lt', Date + ?DATE_RANGE}},
+	Q = {created_at, {'$gt', Date, '$lte', Date + ?DATE_RANGE}},
 	Conn = mongo_pool:get(?POOLNAME),
 	mongo:do(safe, master, Conn, ?DBNAME, fun() ->
 		% 1: cost lots of memory even close the cursor
@@ -122,7 +122,7 @@ do_load_torrents(Date, Size) ->
 		%Cursor = mongo:find(?COLLNAME, {}, {}, Skip),
 		%mongo_cursor:take(Cursor, Size),
 		% 3:
-		Cursor = mongo:find(?COLLNAME, Q, {}),
+		Cursor = mongo:find(?COLLNAME, Q, {}, 0, Size),
 		Ret = mongo_cursor:take(Cursor, Size),
 		mongo_cursor:close(Cursor),
 		Ret