NOTE: rewrite hash_reader, config changed, dht_hash database changed, require to remove existed dht_hash database
Kevin Lynx
11 years ago
parent
75b3d82f4c
commit
dcf0181839
@ -0,0 +1,159 @@
|
||||
%%
|
||||
%% hash_download.erl
|
||||
%% Kevin Lynx
|
||||
%% 07.21.2013
|
||||
%%
|
||||
-module(hash_download).
|
||||
-include("vlog.hrl").
|
||||
-behaviour(gen_server).
|
||||
-export([init/1,
|
||||
handle_call/3,
|
||||
handle_cast/2,
|
||||
handle_info/2,
|
||||
terminate/2,
|
||||
code_change/3]).
|
||||
-export([start_link/1]).
|
||||
-record(state, {dbpool, downloader, downloading = 0, max}).
|
||||
-define(WAIT_TIME, 1*60*1000).
|
||||
|
||||
start_link(DBPool) ->
|
||||
gen_server:start_link(?MODULE, [DBPool], []).
|
||||
|
||||
init([DBPool]) ->
|
||||
{ok, DownPid} = tor_download:start_link(),
|
||||
tor_download_stats:register(DownPid),
|
||||
Max = config:get(max_download_per_reader, 50),
|
||||
{ok, #state{dbpool = DBPool, downloader = DownPid, max = Max}, 0}.
|
||||
|
||||
terminate(_, State) ->
|
||||
{ok, State}.
|
||||
|
||||
code_change(_, _, State) ->
|
||||
{ok, State}.
|
||||
|
||||
handle_cast({process_hash, Doc}, State) ->
|
||||
#state{downloading = DownCnt, downloader = DownPid, max = Max} = State,
|
||||
{BHash} = bson:lookup('_id', Doc),
|
||||
Hash = binary_to_list(BHash),
|
||||
ReqCnt = hash_reader_common:get_req_cnt(Doc),
|
||||
Conn = db_conn(State),
|
||||
AddDown = case db_store_mongo:inc_announce(Conn, Hash, ReqCnt) of
|
||||
true ->
|
||||
?T(?FMT("hash ~s already exists in db", [Hash])),
|
||||
hash_reader_common:on_updated(Conn),
|
||||
0;
|
||||
false ->
|
||||
schedule_download(Conn, DownPid, Hash)
|
||||
end,
|
||||
case AddDown + DownCnt < Max of
|
||||
true ->
|
||||
schedule_next();
|
||||
false ->
|
||||
?T(?FMT("reached the max download ~p, wait", [Max])),
|
||||
wait_downloader_notify
|
||||
end,
|
||||
{noreply, State#state{downloading = DownCnt + AddDown}};
|
||||
|
||||
handle_cast(stop, State) ->
|
||||
{stop, normal, State}.
|
||||
|
||||
handle_call(_, _From, State) ->
|
||||
{noreply, State}.
|
||||
|
||||
handle_info({got_torrent, failed, _Hash}, State) ->
|
||||
#state{downloading = D} = State,
|
||||
schedule_next(),
|
||||
hash_reader_stats:handle_download_failed(),
|
||||
{noreply, State#state{downloading = D - 1}};
|
||||
|
||||
handle_info({got_torrent, ok, Hash, Content}, State) ->
|
||||
schedule_next(),
|
||||
Conn = db_conn(State),
|
||||
true = is_binary(Content),
|
||||
SaveTor = config:get(save_torrent, true),
|
||||
if SaveTor -> loc_torrent_cache:save(Conn, Hash, Content); true -> ok end,
|
||||
NewState = got_torrent_content(State, Hash, Content),
|
||||
hash_reader_stats:handle_download_ok(),
|
||||
{noreply, NewState};
|
||||
|
||||
handle_info({got_torrent_from_cache, Hash, Content}, State) ->
|
||||
on_used_cache(),
|
||||
schedule_next(),
|
||||
NewState = got_torrent_content(State, Hash, Content),
|
||||
{noreply, NewState};
|
||||
|
||||
handle_info(timeout, State) ->
|
||||
schedule_next(),
|
||||
{noreply, State}.
|
||||
|
||||
schedule_next() ->
|
||||
case hash_download_cache:get_one() of
|
||||
{} ->
|
||||
timer:send_after(?WAIT_TIME);
|
||||
Doc ->
|
||||
gen_server:cast(self(), {process_hash, Doc})
|
||||
end.
|
||||
|
||||
schedule_download(Conn, Pid, Hash) ->
|
||||
TryFilter = config:get(check_cache, false),
|
||||
Down = case TryFilter of
|
||||
true ->
|
||||
db_hash_index:exist(Conn, Hash);
|
||||
false ->
|
||||
true
|
||||
end,
|
||||
try_download(Down, Conn, Pid, Hash).
|
||||
|
||||
try_download(false, _, _, Hash) ->
|
||||
?T(?FMT("hash does not exist in index_cache, filter it ~s", [Hash])),
|
||||
0;
|
||||
try_download(true, Conn, Pid, Hash) ->
|
||||
case loc_torrent_cache:load(Conn, Hash) of
|
||||
not_found ->
|
||||
tor_download:download(Pid, Hash);
|
||||
Content ->
|
||||
?T(?FMT("found torrent in local cache ~s", [Hash])),
|
||||
self() ! {got_torrent_from_cache, Hash, Content}
|
||||
end,
|
||||
1.
|
||||
|
||||
db_conn(State) ->
|
||||
#state{dbpool = DBPool} = State,
|
||||
mongo_pool:get(DBPool).
|
||||
|
||||
got_torrent_content(State, MagHash, Content) ->
|
||||
#state{downloading = D} = State,
|
||||
case catch(torrent_file:parse(Content)) of
|
||||
{'EXIT', _} ->
|
||||
?W(?FMT("parse a torrent failed ~s", [MagHash])),
|
||||
skip;
|
||||
{Type, Info} ->
|
||||
got_torrent(State, MagHash, Type, Info)
|
||||
end,
|
||||
State#state{downloading = D - 1}.
|
||||
|
||||
got_torrent(State, Hash, single, {Name, Length}) ->
|
||||
try_save(State, Hash, Name, Length, []);
|
||||
|
||||
got_torrent(State, Hash, multi, {Root, Files}) ->
|
||||
try_save(State, Hash, Root, 0, Files).
|
||||
|
||||
try_save(State, Hash, Name, Length, Files) ->
|
||||
Conn = db_conn(State),
|
||||
case catch db_store_mongo:insert(Conn, Hash, Name, Length, Files) of
|
||||
{'EXIT', Reason} ->
|
||||
?E(?FMT("save torrent failed ~p", [Reason]));
|
||||
_ ->
|
||||
on_saved(Conn)
|
||||
end.
|
||||
|
||||
on_used_cache() ->
|
||||
hash_reader_stats:handle_used_cache().
|
||||
|
||||
on_saved(Conn) ->
|
||||
% `get_peers' here means we have processed a request
|
||||
db_system:stats_get_peers(Conn),
|
||||
% increase the `new' counter
|
||||
db_system:stats_new_saved(Conn),
|
||||
hash_reader_stats:handle_insert().
|
||||
|
||||
@ -0,0 +1,101 @@
|
||||
%%
|
||||
%% hash_download_cache.erl
|
||||
%% Kevin Lynx
|
||||
%% cache these wait_download hashes, the downloader will read hashes from here,
|
||||
%% to avoid database operation, if the cache is too big, save it then.
|
||||
%% 07.21.2013
|
||||
%%
|
||||
-module(hash_download_cache).
|
||||
-include("vlog.hrl").
|
||||
-include("db_common.hrl").
|
||||
-behaviour(gen_server).
|
||||
-export([init/1,
|
||||
handle_call/3,
|
||||
handle_cast/2,
|
||||
handle_info/2,
|
||||
terminate/2,
|
||||
code_change/3]).
|
||||
-export([start_link/1,
|
||||
stop/0,
|
||||
insert/1,
|
||||
get_one/0]).
|
||||
-record(state, {cache = [], max, dbpool}).
|
||||
-define(SAVE_BATCH, 100).
|
||||
|
||||
start_link(DBPool) ->
|
||||
Max = config:get(max_download_cache, 100),
|
||||
gen_server:start_link({local, srv_name()}, ?MODULE, [DBPool, Max], []).
|
||||
|
||||
stop() ->
|
||||
gen_server:cast(srv_name(), stop).
|
||||
|
||||
insert(Doc) ->
|
||||
gen_server:cast(srv_name(), {insert, Doc}).
|
||||
|
||||
get_one() ->
|
||||
gen_server:call(srv_name(), get_one, infinity).
|
||||
|
||||
srv_name() ->
|
||||
?MODULE.
|
||||
|
||||
%
|
||||
init([DBPool, Max]) ->
|
||||
{ok, #state{max = Max, dbpool = DBPool}}.
|
||||
|
||||
terminate(_, State) ->
|
||||
#state{dbpool = DBPool, cache = Cache} = State,
|
||||
check_save(DBPool, Cache, 0),
|
||||
{ok, State}.
|
||||
|
||||
code_change(_, _, State) ->
|
||||
{ok, State}.
|
||||
|
||||
handle_cast({insert, Doc}, State) ->
|
||||
#state{dbpool = DBPool, cache = Cache, max = Max} = State,
|
||||
NewCache = check_save(DBPool, [Doc|Cache], Max),
|
||||
{noreply, State#state{cache = NewCache}};
|
||||
|
||||
handle_cast(stop, State) ->
|
||||
{stop, normal, State}.
|
||||
|
||||
handle_call(get_one, _From, State) ->
|
||||
#state{dbpool = DBPool, cache = Cache} = State,
|
||||
{Doc, NewCache} = try_load(DBPool, Cache),
|
||||
{reply, Doc, State#state{cache = NewCache}};
|
||||
|
||||
handle_call(_, _From, State) ->
|
||||
{noreply, State}.
|
||||
|
||||
handle_info(_, State) ->
|
||||
{noreply, State}.
|
||||
|
||||
check_save(DBPool, Cache, Max) when length(Cache) >= Max ->
|
||||
SplitCnt = 2 * Max div 3,
|
||||
{Remain, ToSave} = lists:split(SplitCnt, Cache),
|
||||
?T(?FMT("download_cache reached the max, save 1/3 ~p", [length(ToSave)])),
|
||||
do_save(DBPool, ToSave),
|
||||
Remain;
|
||||
check_save(_, Cache, _) ->
|
||||
Cache.
|
||||
|
||||
do_save(_DBPool, []) ->
|
||||
ok;
|
||||
do_save(DBPool, Docs) ->
|
||||
Insert = fun(Doc) ->
|
||||
Conn = mongo_pool:get(DBPool),
|
||||
{ID} = bson:lookup('_id', Doc),
|
||||
Sel = {'_id', ID},
|
||||
mongo:do(safe, master, Conn, ?HASH_DBNAME, fun() ->
|
||||
mongo:update(?HASH_DOWNLOAD_COLL, Sel, Doc, true)
|
||||
end)
|
||||
end,
|
||||
[Insert(Doc) || Doc <- Docs].
|
||||
|
||||
try_load(DBPool, []) ->
|
||||
?T("download_cache empty, load hash from db"),
|
||||
Conn = mongo_pool:get(DBPool),
|
||||
{Doc} = hash_reader_common:load_delete_doc(Conn, ?HASH_DOWNLOAD_COLL),
|
||||
{Doc, []};
|
||||
try_load(_, [First|Rest]) ->
|
||||
{First, Rest}.
|
||||
|
||||
@ -0,0 +1,71 @@
|
||||
%%
|
||||
%% hash_reader.erl
|
||||
%% Kevin Lynx
|
||||
%% 07.21.2013
|
||||
%%
|
||||
-module(hash_reader2).
|
||||
-include("vlog.hrl").
|
||||
-include("db_common.hrl").
|
||||
-behaviour(gen_server).
|
||||
-export([init/1,
|
||||
handle_call/3,
|
||||
handle_cast/2,
|
||||
handle_info/2,
|
||||
terminate/2,
|
||||
code_change/3]).
|
||||
-export([start_link/1]).
|
||||
-define(WAIT_TIME, 1*60*1000).
|
||||
-record(state, {dbpool}).
|
||||
|
||||
start_link(DBPool) ->
|
||||
gen_server:start_link(?MODULE, [DBPool], []).
|
||||
|
||||
init([DBPool]) ->
|
||||
hash_download:start_link(DBPool),
|
||||
{ok, #state{dbpool = DBPool}, 0}.
|
||||
|
||||
terminate(_, State) ->
|
||||
{ok, State}.
|
||||
|
||||
code_change(_, _, State) ->
|
||||
{ok, State}.
|
||||
|
||||
handle_cast({process_hash, Doc}, State) ->
|
||||
{BHash} = bson:lookup('_id', Doc),
|
||||
Hash = binary_to_list(BHash),
|
||||
ReqCnt = hash_reader_common:get_req_cnt(Doc),
|
||||
Conn = db_conn(State),
|
||||
case db_store_mongo:inc_announce(Conn, Hash, ReqCnt) of
|
||||
true ->
|
||||
hash_reader_common:on_updated(Conn);
|
||||
false ->
|
||||
?T(?FMT("insert doc ~s to download_cache", [Hash])),
|
||||
hash_download_cache:insert(Doc)
|
||||
end,
|
||||
schedule_next(Conn),
|
||||
{noreply, State};
|
||||
|
||||
handle_cast(stop, State) ->
|
||||
{stop, normal, State}.
|
||||
|
||||
handle_call(_, _From, State) ->
|
||||
{noreply, State}.
|
||||
|
||||
handle_info(timeout, State) ->
|
||||
Conn = db_conn(State),
|
||||
schedule_next(Conn),
|
||||
{noreply, State}.
|
||||
|
||||
schedule_next(Conn) ->
|
||||
case hash_reader_common:load_delete_doc(Conn, ?HASH_COLLNAME) of
|
||||
{} ->
|
||||
?T("start to wait for new hash"),
|
||||
timer:send_after(?WAIT_TIME, timeout);
|
||||
{Doc} ->
|
||||
gen_server:cast(self(), {process_hash, Doc})
|
||||
end.
|
||||
|
||||
db_conn(State) ->
|
||||
#state{dbpool = DBPool} = State,
|
||||
mongo_pool:get(DBPool).
|
||||
|
||||
@ -0,0 +1,34 @@
|
||||
%%
|
||||
%% hash_reader_common.erl
|
||||
%% Kevin Lynx
|
||||
%% 07.21.2013
|
||||
%%
|
||||
-module(hash_reader_common).
|
||||
-include("db_common.hrl").
|
||||
-export([get_req_cnt/1,
|
||||
on_updated/1,
|
||||
load_delete_doc/2]).
|
||||
|
||||
get_req_cnt(Doc) ->
|
||||
case bson:lookup(req_cnt, Doc) of
|
||||
{} -> 0;
|
||||
{R} -> R
|
||||
end.
|
||||
|
||||
on_updated(Conn) ->
|
||||
% `get_peers' here means we have processed a request
|
||||
db_system:stats_get_peers(Conn),
|
||||
% also increase the updated counter
|
||||
db_system:stats_updated(Conn),
|
||||
hash_reader_stats:handle_update().
|
||||
|
||||
load_delete_doc(Conn, Col) ->
|
||||
Cmd = {findAndModify, Col, fields, {}, remove, true},
|
||||
Ret = mongo:do(safe, master, Conn, ?HASH_DBNAME, fun() ->
|
||||
mongo:command(Cmd)
|
||||
end),
|
||||
case Ret of
|
||||
{value, undefined, ok, 1.0} -> {};
|
||||
{value, Obj, lastErrorObject, _, ok, 1.0} -> {Obj}
|
||||
end.
|
||||
|
||||
Loading…
Reference in New Issue