diff --git a/sshagent/server.py b/sshagent/server.py
index 1e741f5..38da6c5 100644
--- a/sshagent/server.py
+++ b/sshagent/server.py
@@ -45,9 +45,8 @@ def handle_connection(conn, handler):
         raise
 
 
-def server_thread(server, keys, signer):
+def server_thread(server, handler):
     log.debug('server thread started')
-    handler = protocol.Handler(keys=keys, signer=signer)
     while True:
         log.debug('waiting for connection on %s', server.getsockname())
         try:
@@ -76,7 +75,8 @@ def serve(key_files, signer, sock_path=None):
     keys = [formats.parse_public_key(k) for k in key_files]
     environ = {'SSH_AUTH_SOCK': sock_path, 'SSH_AGENT_PID': str(os.getpid())}
     with unix_domain_socket_server(sock_path) as server:
-        with spawn(server_thread, server=server, keys=keys, signer=signer):
+        handler = protocol.Handler(keys=keys, signer=signer)
+        with spawn(server_thread, server=server, handler=handler):
             try:
                 yield environ
             finally: