'live_grep' freezes on large text files (#211)

main
bhagwan 3 years ago
parent cb526dbb0c
commit e02451600a

@ -213,6 +213,15 @@ end
M.run_builtin = function(selected)
local method = selected[1]
vim.cmd(string.format("lua require'fzf-lua'.%s()", method))
-- not sure what is causing this, tested with
-- 'NVIM v0.6.0-dev+575-g2ef9d2a66'
-- vim.cmd("startinsert") doesn't start INSERT mode
-- 'mode' returns { blocking = false, mode = "t" }
-- manually input 'i' seems to workaround this issue
local mode = vim.api.nvim_get_mode()
if mode.mode and mode.mode ~= 'i' then
vim.cmd[[noautocmd lua vim.api.nvim_feedkeys('i', 'n', true)]]
end
end
M.ex_run = function(selected)

@ -238,7 +238,8 @@ M.globals.grep = {
actions = M.globals.files.actions,
-- live_grep_glob options
glob_flag = "--iglob", -- for case sensitive globs use '--glob'
glob_separator = "%s%-%-" -- query separator pattern (lua): ' --'
glob_separator = "%s%-%-", -- query separator pattern (lua): ' --'
data_limit = 128 * 1024, -- 'live_grep' libuv chunk data limit
}
M.globals.args = {
previewer = M._default_previewer_fn,
@ -546,7 +547,10 @@ function M.normalize_opts(opts, defaults)
-- Merge required tables from globals
for _, k in ipairs({ 'winopts', 'keymap', 'fzf_opts', 'previewers' }) do
opts[k] = vim.tbl_deep_extend("keep", opts[k] or {}, M.globals[k] or {})
opts[k] = vim.tbl_deep_extend("keep",
-- must clone or map will be saved as reference
-- and then overwritten if found in 'backward_compat'
opts[k] or {}, utils.tbl_deep_clone(M.globals[k]) or {})
end
-- backward compatibility, rhs overrides lhs

@ -6,6 +6,7 @@ local config = require "fzf-lua.config"
local actions = require "fzf-lua.actions"
local win = require "fzf-lua.win"
local libuv = require "fzf-lua.libuv"
local shell = require "fzf-lua.shell"
local M = {}
@ -370,7 +371,7 @@ M.set_fzf_interactive_cb = function(opts)
local placeholder = utils._if(opts._is_skim, '"{}"', '{q}')
local uv = vim.loop
local raw_async_act = require("fzf.actions").raw_async_action(function(pipe, args)
local raw_async_act = shell.raw_async_action(function(pipe, args)
coroutine.wrap(function()

@ -148,6 +148,7 @@ local _modules = {
'path',
'utils',
'libuv',
'shell',
'config',
'actions',
}

@ -1,6 +1,5 @@
local shell = require "fzf-lua.shell"
local utils = require "fzf-lua.utils"
local async_action = require("fzf.actions").async_action
local raw_async_action = require("fzf.actions").raw_async_action
local uv = vim.loop
local M = {}
@ -17,6 +16,14 @@ local function find_last_newline(str)
end
end
local function find_next_newline(str, start_idx)
for i=start_idx or 1,#str do
if string_byte(str, i) == 10 then
return i
end
end
end
local function coroutine_callback(fn)
local co = coroutine.running()
local callback = function(...)
@ -49,11 +56,10 @@ M.spawn = function(opts, fn_transform, fn_done)
local error_pipe = uv.new_pipe(false)
local write_cb_count = 0
local prev_line_content = nil
local num_lines = 0
if opts.fn_transform then fn_transform = opts.fn_transform end
local shell = vim.env.SHELL or "sh"
local finish = function(sig, pid)
output_pipe:shutdown()
error_pipe:shutdown()
@ -68,7 +74,7 @@ M.spawn = function(opts, fn_transform, fn_done)
-- https://github.com/luvit/luv/blob/master/docs.md
-- uv.spawn returns tuple: handle, pid
local _, pid = uv.spawn(shell, {
local _, pid = uv.spawn(vim.env.SHELL or "sh", {
args = { "-c", opts.cmd },
stdio = { nil, output_pipe, error_pipe },
cwd = opts.cwd
@ -105,13 +111,43 @@ M.spawn = function(opts, fn_transform, fn_done)
end)
end
local function process_lines(str)
write_cb(str:gsub("[^\n]+",
local function process_lines(data)
if opts.data_limit and opts.data_limit > 0 and #data>opts.data_limit then
vim.defer_fn(function()
utils.warn(("received large data chunk (%db), consider adding '--max-columns=512' to ripgrep flags\nDATA: '%s'")
:format(#data, utils.strip_ansi_coloring(data):sub(1,80)))
end, 0)
end
write_cb(data:gsub("[^\n]+",
function(x)
return fn_transform(x)
end))
end
--[[ local function process_lines(data)
if opts.data_limit and opts.data_limit > 0 and #data>opts.data_limit then
vim.defer_fn(function()
utils.warn(("received large data chunk (%db, consider adding '--max-columns=512' to ripgrep flags\nDATA: '%s'")
:format(#data, utils.strip_ansi_coloring(data):sub(1,80)))
end, 0)
end
local start_idx = 1
repeat
num_lines = num_lines + 1
local nl_idx = find_next_newline(data, start_idx)
local line = data:sub(start_idx, nl_idx)
if #line > 1024 then
vim.defer_fn(function()
utils.warn(("long line %d bytes, '%s'")
:format(#line, utils.strip_ansi_coloring(line):sub(1,60)))
end, 0)
line = line:sub(1,512) .. '\n'
end
write_cb(fn_transform(line))
start_idx = nl_idx + 1
until start_idx >= #data
end --]]
local read_cb = function(err, data)
if err then
@ -134,7 +170,11 @@ M.spawn = function(opts, fn_transform, fn_done)
else
local nl_index = find_last_newline(data)
if not nl_index then
prev_line_content = data
-- chunk size is 64K, limit previous line length to 1K
-- max line length is therefor 1K + 64K (leftover + full chunk)
-- without this we can memory fault on extremely long lines (#185)
-- or have UI freezes (#211)
prev_line_content = data:sub(1, 1024)
else
prev_line_content = string_sub(data, nl_index + 1)
local stripped_with_newline = string_sub(data, 1, nl_index)
@ -194,7 +234,7 @@ end
M.spawn_nvim_fzf_action = function(fn, fzf_field_expression)
return async_action(function(pipe, ...)
return shell.async_action(function(pipe, ...)
local function on_finish(_, _)
if pipe and not uv.is_closing(pipe) then
@ -224,7 +264,7 @@ M.spawn_reload_cmd_action = function(opts, fzf_field_expression)
local _pid = nil
return raw_async_action(function(pipe, args)
return shell.raw_async_action(function(pipe, args)
local function on_pid(pid)
_pid = pid
@ -258,7 +298,10 @@ M.spawn_reload_cmd_action = function(opts, fzf_field_expression)
cb_finish = on_finish,
cb_write = on_write,
cb_pid = on_pid,
}, opts._fn_transform)
data_limit = opts.data_limit,
-- must send false, 'coroutinify' adds callback as last argument
-- which will conflict with the 'fn_transform' argument
}, opts._fn_transform or false)
end, fzf_field_expression)
end

@ -1,7 +1,7 @@
local path = require "fzf-lua.path"
local shell = require "fzf-lua.shell"
local utils = require "fzf-lua.utils"
local previewer_base = require "fzf-lua.previewer"
local raw_action = require("fzf.actions").raw_action
local api = vim.api
local fn = vim.fn
@ -169,7 +169,7 @@ function Previewer.base:display_entry(entry_str)
end
function Previewer.base:action(_)
local act = raw_action(function (items, _, _)
local act = shell.raw_action(function (items, _, _)
self:display_entry(items[1])
return ""
end, "{}")

@ -1,8 +1,8 @@
local path = require "fzf-lua.path"
local shell = require "fzf-lua.shell"
local utils = require "fzf-lua.utils"
local libuv = require "fzf-lua.libuv"
local previewer_base = require "fzf-lua.previewer"
local raw_action = require("fzf.actions").raw_action
local Previewer = {}
Previewer.base = {}
@ -60,7 +60,7 @@ function Previewer.cmd:action(o)
if self.opts._line_placeholder then
filespec = "{1}"
end
local act = raw_action(function (items, fzf_lines, _)
local act = shell.raw_action(function (items, fzf_lines, _)
-- only preview first item
local file = path.entry_to_file(items[1], not self.relative and self.opts.cwd)
return file.path

@ -2,9 +2,9 @@ if not pcall(require, "fzf") then
return
end
local action = require("fzf.actions").action
local core = require "fzf-lua.core"
local path = require "fzf-lua.path"
local shell = require "fzf-lua.shell"
local utils = require "fzf-lua.utils"
local config = require "fzf-lua.config"
local actions = require "fzf-lua.actions"
@ -153,7 +153,7 @@ M.buffers = function(opts)
opts = config.normalize_opts(opts, config.globals.buffers)
if not opts then return end
local act = action(function (items, fzf_lines, _)
local act = shell.action(function (items, fzf_lines, _)
-- only preview first item
local item = items[1]
local buf = getbufnumber(item)

@ -2,9 +2,8 @@ if not pcall(require, "fzf") then
return
end
local action = require("fzf.actions").action
local core = require "fzf-lua.core"
local utils = require "fzf-lua.utils"
local shell = require "fzf-lua.shell"
local config = require "fzf-lua.config"
local actions = require "fzf-lua.actions"
@ -24,7 +23,7 @@ M.colorschemes = function(opts)
if not opts then return end
coroutine.wrap(function ()
local prev_act = action(function (args)
local prev_act = shell.action(function (args)
if opts.live_preview and args then
local colorscheme = args[1]
vim.cmd("colorscheme " .. colorscheme)

@ -4,6 +4,7 @@ end
local core = require "fzf-lua.core"
local utils = require "fzf-lua.utils"
local shell = require "fzf-lua.shell"
local config = require "fzf-lua.config"
local libuv = require "fzf-lua.libuv"
@ -63,7 +64,7 @@ M.files_resume = function(opts)
return
end
local raw_act = require("fzf.actions").raw_action(function(args)
local raw_act = shell.raw_action(function(args)
last_query = args[1]
end, "{q}")

@ -2,13 +2,10 @@ if not pcall(require, "fzf") then
return
end
local raw_action = require("fzf.actions").raw_action
local raw_async_action = require("fzf.actions").raw_async_action
local core = require "fzf-lua.core"
local utils = require "fzf-lua.utils"
local config = require "fzf-lua.config"
local actions = require "fzf-lua.actions"
local uv = vim.loop
local M = {}

@ -2,8 +2,8 @@ if not pcall(require, "fzf") then
return
end
local action = require("fzf.actions").action
local core = require "fzf-lua.core"
local shell = require "fzf-lua.shell"
local config = require "fzf-lua.config"
local actions = require "fzf-lua.actions"
@ -18,7 +18,7 @@ M.metatable = function(opts)
coroutine.wrap(function ()
local prev_act = action(function (args)
local prev_act = shell.action(function (args)
-- TODO: retreive method help
local help = ''
return string.format("%s:%s", args[1], help)

@ -2,9 +2,9 @@ if not pcall(require, "fzf") then
return
end
local action = require("fzf.actions").action
local core = require "fzf-lua.core"
local utils = require "fzf-lua.utils"
local shell = require "fzf-lua.shell"
local config = require "fzf-lua.config"
local actions = require "fzf-lua.actions"
@ -19,7 +19,7 @@ M.commands = function(opts)
local commands = vim.api.nvim_get_commands {}
local prev_act = action(function (args)
local prev_act = shell.action(function (args)
local cmd = args[1]
if commands[cmd] then
cmd = vim.inspect(commands[cmd])
@ -100,7 +100,7 @@ M.marks = function(opts)
local marks = vim.fn.execute("marks")
marks = vim.split(marks, "\n")
local prev_act = action(function (args, fzf_lines, _)
local prev_act = shell.action(function (args, fzf_lines, _)
local mark = args[1]:match("[^ ]+")
local bufnr, lnum, _, _ = unpack(vim.fn.getpos("'"..mark))
if vim.api.nvim_buf_is_loaded(bufnr) then
@ -154,7 +154,7 @@ M.registers = function(opts)
table.insert(registers, string.char(i))
end
local prev_act = action(function (args)
local prev_act = shell.action(function (args)
local r = args[1]:match("%[(.*)%] ")
local _, contents = pcall(vim.fn.getreg, r)
return contents or args[1]
@ -216,7 +216,7 @@ M.keymaps = function(opts)
end
end
local prev_act = action(function (args)
local prev_act = shell.action(function (args)
local k = args[1]:match("(%[.*%]) ")
local v = keymaps[k]
if v then

@ -0,0 +1,112 @@
-- for testing, copied from:
-- https://github.com/vijaymarupudi/nvim-fzf/blob/master/lua/fzf/actions.lua
local uv = vim.loop
local path = require "fzf-lua.path"
local M = {}
local _counter = 0
local _registry = {}
function M.register_func(fn)
_counter = _counter + 1
_registry[_counter] = fn
return _counter
end
function M.get_func(counter)
return _registry[counter]
end
-- creates a new address to listen to messages from actions. This is important,
-- if the user is using a custom fixed $NVIM_LISTEN_ADDRESS. Different neovim
-- instances will then use the same path as the address and it causes a mess,
-- i.e. actions stop working on the old instance. So we create our own (random
-- path) RPC server for this instance if it hasn't been started already.
local action_server_address = nil
function M.raw_async_action(fn, fzf_field_expression)
if not fzf_field_expression then
fzf_field_expression = "{+}"
end
local receiving_function = function(pipe_path, ...)
local pipe = uv.new_pipe(false)
local args = {...}
uv.pipe_connect(pipe, pipe_path, function(err)
vim.schedule(function ()
fn(pipe, unpack(args))
end)
end)
end
if not action_server_address then
action_server_address = vim.fn.serverstart()
end
local id = M.register_func(receiving_function)
-- this is for windows WSL and AppImage users, their nvim path isn't just
-- 'nvim', it can be something else
local nvim_command = vim.v.argv[1]
local action_string = string.format("%s --headless --clean --cmd %s %s %s %s",
vim.fn.shellescape(nvim_command),
vim.fn.shellescape("luafile " .. path.join{vim.g.fzf_lua_directory, "shell_helper.lua"}),
vim.fn.shellescape(action_server_address),
id,
fzf_field_expression)
return action_string, id
end
function M.async_action(fn, fzf_field_expression)
local action_string, id = M.raw_async_action(fn, fzf_field_expression)
return vim.fn.shellescape(action_string), id
end
function M.raw_action(fn, fzf_field_expression)
local receiving_function = function(pipe, ...)
local ret = fn(...)
local on_complete = function(err)
-- We are NOT asserting, in case fzf closes the pipe before we can send
-- the preview.
-- assert(not err)
uv.close(pipe)
end
if type(ret) == "string" then
uv.write(pipe, ret, on_complete)
elseif type(ret) == nil then
on_complete()
elseif type(ret) == "table" then
if not vim.tbl_isempty(ret) then
uv.write(pipe, vim.tbl_map(function(x) return x.."\n" end, ret), on_complete)
else
on_complete()
end
else
uv.write(pipe, tostring(ret) .. "\n", on_complete)
end
end
return M.raw_async_action(receiving_function, fzf_field_expression)
end
function M.action(fn, fzf_field_expression)
local action_string, id = M.raw_action(fn, fzf_field_expression)
return vim.fn.shellescape(action_string), id
end
-- set to 'true' to use 'nvim-fzf'
-- set to 'false' for debugging using the local version
if false then
M.action = require("fzf.actions").action
M.raw_action = require("fzf.actions").raw_action
M.async_action = require("fzf.actions").async_action
M.raw_async_action = require("fzf.actions").raw_async_action
end
return M

@ -0,0 +1,69 @@
-- for testing, copied from:
-- https://github.com/vijaymarupudi/nvim-fzf/blob/master/action_helper.lua
local uv = vim.loop
local function get_preview_socket()
local tmp = vim.fn.tempname()
local socket = uv.new_pipe(false)
uv.pipe_bind(socket, tmp)
return socket, tmp
end
local preview_socket, preview_socket_path = get_preview_socket()
uv.listen(preview_socket, 100, function(err)
local preview_receive_socket = uv.new_pipe(false)
-- start listening
uv.accept(preview_socket, preview_receive_socket)
preview_receive_socket:read_start(function(err, data)
assert(not err)
if not data then
uv.close(preview_receive_socket)
uv.close(preview_socket)
vim.schedule(function()
vim.cmd[[qall]]
end)
return
end
io.write(data)
end)
end)
local function_id = tonumber(vim.fn.argv(1))
local success, errmsg = pcall(function ()
local nargs = vim.fn.argc()
local args = {}
-- this is guaranteed to be 2 or more, we are interested in those greater than 2
for i=3,nargs do
-- vim uses zero indexing
table.insert(args, vim.fn.argv(i - 1))
end
local environ = vim.fn.environ()
local chan_id = vim.fn.sockconnect("pipe", vim.fn.argv(0), { rpc = true })
-- for skim compatibility
local preview_lines = environ.FZF_PREVIEW_LINES or environ.LINES
local preview_cols = environ.FZF_PREVIEW_COLUMNS or environ.COLUMNS
vim.rpcrequest(chan_id, "nvim_exec_lua", [[
local luaargs = {...}
local function_id = luaargs[1]
local preview_socket_path = luaargs[2]
local fzf_selection = luaargs[3]
local fzf_lines = luaargs[4]
local fzf_columns = luaargs[5]
local usr_func = require"fzf-lua.shell".get_func(function_id)
return usr_func(preview_socket_path, fzf_selection, fzf_lines, fzf_columns)
]], {
function_id,
preview_socket_path,
args,
tonumber(preview_lines),
tonumber(preview_cols)
})
vim.fn.chanclose(chan_id)
end)
if not success then
io.write("FzfLua Error:\n\n" .. errmsg .. "\n")
vim.cmd [[qall]]
end
Loading…
Cancel
Save