mirror of https://github.com/koreader/koreader
add table persistence module
parent
47b0d4089a
commit
b8f0dc3752
@ -1 +1 @@
|
||||
Subproject commit dec3df4e5a96d7a039b8e6aeac4000c34c76a83a
|
||||
Subproject commit 22d3b504d43e058a0d06c585fda8459749de0c3e
|
@ -0,0 +1,150 @@
|
||||
local bitser = require("ffi/bitser")
|
||||
local dump = require("dump")
|
||||
local lfs = require("libs/libkoreader-lfs")
|
||||
local logger = require("logger")
|
||||
|
||||
local function readFile(file, bytes)
|
||||
local f, str, err
|
||||
f, err = io.open(file, "rb")
|
||||
if not f then
|
||||
return nil, err
|
||||
end
|
||||
str, err = f:read(bytes or "*a")
|
||||
f:close()
|
||||
if not str then
|
||||
return nil, err
|
||||
end
|
||||
return str
|
||||
end
|
||||
|
||||
local codecs = {
|
||||
-- bitser: binary form, fast encode/decode, low size. Not human readable.
|
||||
bitser = {
|
||||
id = "bitser",
|
||||
reads_from_file = false,
|
||||
|
||||
serialize = function(t)
|
||||
local ok, str = pcall(bitser.dumps, t)
|
||||
if not ok then
|
||||
return nil, "cannot serialize " .. tostring(t)
|
||||
end
|
||||
return str
|
||||
end,
|
||||
|
||||
deserialize = function(str)
|
||||
local ok, t = pcall(bitser.loads, str)
|
||||
if not ok then
|
||||
return nil, "malformed serialized data"
|
||||
end
|
||||
return t
|
||||
end,
|
||||
},
|
||||
-- dump: human readable, pretty printed, fast enough for most user cases.
|
||||
dump = {
|
||||
id = "dump",
|
||||
reads_from_file = true,
|
||||
|
||||
serialize = function(t, as_bytecode)
|
||||
local content
|
||||
if as_bytecode then
|
||||
local bytecode, err = load("return " .. dump(t))
|
||||
if not bytecode then
|
||||
logger.warn("cannot convert table to bytecode", err, "fallback to text")
|
||||
else
|
||||
content = string.dump(bytecode, true)
|
||||
end
|
||||
end
|
||||
if not content then
|
||||
content = "return " .. dump(t)
|
||||
end
|
||||
return content
|
||||
end,
|
||||
|
||||
deserialize = function(str)
|
||||
local t, err = loadfile(str)
|
||||
if not t then
|
||||
t, err = loadstring(str)
|
||||
end
|
||||
if not t then
|
||||
return nil, err
|
||||
end
|
||||
return t()
|
||||
end,
|
||||
}
|
||||
}
|
||||
|
||||
local Persist = {}
|
||||
|
||||
function Persist:new(o)
|
||||
o = o or {}
|
||||
assert(type(o.path) == "string", "path is required")
|
||||
o.codec = o.codec or "dump"
|
||||
setmetatable(o, self)
|
||||
self.__index = self
|
||||
return o
|
||||
end
|
||||
|
||||
function Persist:exists()
|
||||
local mode = lfs.attributes(self.path, "mode")
|
||||
if mode then
|
||||
return mode == "file"
|
||||
end
|
||||
end
|
||||
|
||||
function Persist:timestamp()
|
||||
return lfs.attributes(self.path, "modification")
|
||||
end
|
||||
|
||||
function Persist:size()
|
||||
return lfs.attributes(self.path, "size")
|
||||
end
|
||||
|
||||
function Persist:load()
|
||||
local t, err
|
||||
if codecs[self.codec].reads_from_file then
|
||||
t, err = codecs[self.codec].deserialize(self.path)
|
||||
else
|
||||
local str
|
||||
str, err = readFile(self.path)
|
||||
if not str then
|
||||
return nil, err
|
||||
end
|
||||
t, err = codecs[self.codec].deserialize(str)
|
||||
end
|
||||
if not t then
|
||||
return nil, err
|
||||
end
|
||||
return t
|
||||
end
|
||||
|
||||
function Persist:save(t, as_bytecode)
|
||||
local str, file, err
|
||||
str, err = codecs[self.codec].serialize(t, as_bytecode)
|
||||
if not str then
|
||||
return nil, err
|
||||
end
|
||||
file, err = io.open(self.path, "wb")
|
||||
if not file then
|
||||
return nil, err
|
||||
end
|
||||
file:write(str)
|
||||
file:close()
|
||||
return true
|
||||
end
|
||||
|
||||
function Persist:delete()
|
||||
if not self:exists() then return end
|
||||
return os.remove(self.path)
|
||||
end
|
||||
|
||||
function Persist.getCodec(name)
|
||||
local fallback = codecs["dump"]
|
||||
for key, codec in pairs(codecs) do
|
||||
if type(key) == "string" and key == name then
|
||||
return codec
|
||||
end
|
||||
end
|
||||
return fallback
|
||||
end
|
||||
|
||||
return Persist
|
@ -0,0 +1,89 @@
|
||||
describe("Persist module", function()
|
||||
local Persist
|
||||
local sample
|
||||
local bitserInstance, dumpInstance
|
||||
local ser, deser, str, tab
|
||||
local fail = { a = function() end, }
|
||||
|
||||
local function arrayOf(n)
|
||||
assert(type(n) == "number", "wrong type (expected number)")
|
||||
local t = {}
|
||||
for i = 1, n do
|
||||
table.insert(t, i, {
|
||||
a = "sample " .. tostring(i),
|
||||
b = true,
|
||||
c = nil,
|
||||
d = i,
|
||||
e = {
|
||||
f = {
|
||||
g = nil,
|
||||
h = false,
|
||||
},
|
||||
},
|
||||
})
|
||||
end
|
||||
return t
|
||||
end
|
||||
|
||||
setup(function()
|
||||
require("commonrequire")
|
||||
Persist = require("persist")
|
||||
bitserInstance = Persist:new{ path = "test.dat", codec = "bitser" }
|
||||
dumpInstance = Persist:new { path = "test.txt", codec = "dump" }
|
||||
sample = arrayOf(1000)
|
||||
end)
|
||||
|
||||
it("should save a table to file", function()
|
||||
assert.is_true(bitserInstance:save(sample))
|
||||
assert.is_true(dumpInstance:save(sample))
|
||||
end)
|
||||
|
||||
it("should generate a valid file", function()
|
||||
assert.is_true(bitserInstance:exists())
|
||||
assert.is_true(bitserInstance:size() > 0)
|
||||
assert.is_true(type(bitserInstance:timestamp()) == "number")
|
||||
end)
|
||||
|
||||
it("should load a table from file", function()
|
||||
assert.are.same(sample, bitserInstance:load())
|
||||
assert.are.same(sample, dumpInstance:load())
|
||||
end)
|
||||
|
||||
it("should delete the file", function()
|
||||
bitserInstance:delete()
|
||||
dumpInstance:delete()
|
||||
assert.is_nil(bitserInstance:exists())
|
||||
assert.is_nil(dumpInstance:exists())
|
||||
end)
|
||||
|
||||
it("should return standalone serializers/deserializers", function()
|
||||
tab = sample
|
||||
for _, codec in ipairs({"dump", "bitser"}) do
|
||||
assert.is_true(Persist.getCodec(codec).id == codec)
|
||||
ser = Persist.getCodec(codec).serialize
|
||||
deser = Persist.getCodec(codec).deserialize
|
||||
str = ser(tab)
|
||||
assert.are.same(deser(str), tab)
|
||||
str, ser, deser = nil, nil, nil
|
||||
end
|
||||
end)
|
||||
|
||||
it("should work with huge tables", function()
|
||||
tab = arrayOf(100000)
|
||||
ser = Persist.getCodec("bitser").serialize
|
||||
deser = Persist.getCodec("bitser").deserialize
|
||||
str = ser(tab)
|
||||
assert.are.same(deser(str), tab)
|
||||
end)
|
||||
|
||||
it ("should fail to serialize functions", function()
|
||||
for _, codec in ipairs({"dump", "bitser"}) do
|
||||
assert.is_true(Persist.getCodec(codec).id == codec)
|
||||
ser = Persist.getCodec(codec).serialize
|
||||
deser = Persist.getCodec(codec).deserialize
|
||||
str = ser(fail)
|
||||
assert.are_not.same(deser(str), fail)
|
||||
end
|
||||
end)
|
||||
|
||||
end)
|
Loading…
Reference in New Issue