From b8f0dc37524bacbcb9a0d8e6e260042ecd28ee3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mart=C3=ADn=20Fdez?= Date: Wed, 13 Jan 2021 11:45:00 +0100 Subject: [PATCH] add table persistence module --- base | 2 +- frontend/persist.lua | 150 +++++++++++++++++++++++++++++++++++++ frontend/util.lua | 17 ----- spec/unit/persist_spec.lua | 89 ++++++++++++++++++++++ 4 files changed, 240 insertions(+), 18 deletions(-) create mode 100644 frontend/persist.lua create mode 100644 spec/unit/persist_spec.lua diff --git a/base b/base index dec3df4e5..22d3b504d 160000 --- a/base +++ b/base @@ -1 +1 @@ -Subproject commit dec3df4e5a96d7a039b8e6aeac4000c34c76a83a +Subproject commit 22d3b504d43e058a0d06c585fda8459749de0c3e diff --git a/frontend/persist.lua b/frontend/persist.lua new file mode 100644 index 000000000..3bb59cbb3 --- /dev/null +++ b/frontend/persist.lua @@ -0,0 +1,150 @@ +local bitser = require("ffi/bitser") +local dump = require("dump") +local lfs = require("libs/libkoreader-lfs") +local logger = require("logger") + +local function readFile(file, bytes) + local f, str, err + f, err = io.open(file, "rb") + if not f then + return nil, err + end + str, err = f:read(bytes or "*a") + f:close() + if not str then + return nil, err + end + return str +end + +local codecs = { + -- bitser: binary form, fast encode/decode, low size. Not human readable. + bitser = { + id = "bitser", + reads_from_file = false, + + serialize = function(t) + local ok, str = pcall(bitser.dumps, t) + if not ok then + return nil, "cannot serialize " .. tostring(t) + end + return str + end, + + deserialize = function(str) + local ok, t = pcall(bitser.loads, str) + if not ok then + return nil, "malformed serialized data" + end + return t + end, + }, + -- dump: human readable, pretty printed, fast enough for most user cases. + dump = { + id = "dump", + reads_from_file = true, + + serialize = function(t, as_bytecode) + local content + if as_bytecode then + local bytecode, err = load("return " .. dump(t)) + if not bytecode then + logger.warn("cannot convert table to bytecode", err, "fallback to text") + else + content = string.dump(bytecode, true) + end + end + if not content then + content = "return " .. dump(t) + end + return content + end, + + deserialize = function(str) + local t, err = loadfile(str) + if not t then + t, err = loadstring(str) + end + if not t then + return nil, err + end + return t() + end, + } +} + +local Persist = {} + +function Persist:new(o) + o = o or {} + assert(type(o.path) == "string", "path is required") + o.codec = o.codec or "dump" + setmetatable(o, self) + self.__index = self + return o +end + +function Persist:exists() + local mode = lfs.attributes(self.path, "mode") + if mode then + return mode == "file" + end +end + +function Persist:timestamp() + return lfs.attributes(self.path, "modification") +end + +function Persist:size() + return lfs.attributes(self.path, "size") +end + +function Persist:load() + local t, err + if codecs[self.codec].reads_from_file then + t, err = codecs[self.codec].deserialize(self.path) + else + local str + str, err = readFile(self.path) + if not str then + return nil, err + end + t, err = codecs[self.codec].deserialize(str) + end + if not t then + return nil, err + end + return t +end + +function Persist:save(t, as_bytecode) + local str, file, err + str, err = codecs[self.codec].serialize(t, as_bytecode) + if not str then + return nil, err + end + file, err = io.open(self.path, "wb") + if not file then + return nil, err + end + file:write(str) + file:close() + return true +end + +function Persist:delete() + if not self:exists() then return end + return os.remove(self.path) +end + +function Persist.getCodec(name) + local fallback = codecs["dump"] + for key, codec in pairs(codecs) do + if type(key) == "string" and key == name then + return codec + end + end + return fallback +end + +return Persist diff --git a/frontend/util.lua b/frontend/util.lua index 1abc154e2..d52a5d91d 100644 --- a/frontend/util.lua +++ b/frontend/util.lua @@ -1160,23 +1160,6 @@ function util.clearTable(t) for i = 0, c do t[i] = nil end end ---- Dumps a table into a file. ---- @table t the table to be dumped ---- @string file the file to store the table ---- @treturn bool true on success, false otherwise -function util.dumpTable(t, file) - if not t or not file or file == "" then return end - local dump = require("dump") - local f = io.open(file, "w") - if f then - f:write("return "..dump(t)) - f:close() - return true - end - return false -end - - --- Encode URL also known as percent-encoding see https://en.wikipedia.org/wiki/Percent-encoding --- @string text the string to encode --- @treturn encode string diff --git a/spec/unit/persist_spec.lua b/spec/unit/persist_spec.lua new file mode 100644 index 000000000..879a98251 --- /dev/null +++ b/spec/unit/persist_spec.lua @@ -0,0 +1,89 @@ +describe("Persist module", function() + local Persist + local sample + local bitserInstance, dumpInstance + local ser, deser, str, tab + local fail = { a = function() end, } + + local function arrayOf(n) + assert(type(n) == "number", "wrong type (expected number)") + local t = {} + for i = 1, n do + table.insert(t, i, { + a = "sample " .. tostring(i), + b = true, + c = nil, + d = i, + e = { + f = { + g = nil, + h = false, + }, + }, + }) + end + return t + end + + setup(function() + require("commonrequire") + Persist = require("persist") + bitserInstance = Persist:new{ path = "test.dat", codec = "bitser" } + dumpInstance = Persist:new { path = "test.txt", codec = "dump" } + sample = arrayOf(1000) + end) + + it("should save a table to file", function() + assert.is_true(bitserInstance:save(sample)) + assert.is_true(dumpInstance:save(sample)) + end) + + it("should generate a valid file", function() + assert.is_true(bitserInstance:exists()) + assert.is_true(bitserInstance:size() > 0) + assert.is_true(type(bitserInstance:timestamp()) == "number") + end) + + it("should load a table from file", function() + assert.are.same(sample, bitserInstance:load()) + assert.are.same(sample, dumpInstance:load()) + end) + + it("should delete the file", function() + bitserInstance:delete() + dumpInstance:delete() + assert.is_nil(bitserInstance:exists()) + assert.is_nil(dumpInstance:exists()) + end) + + it("should return standalone serializers/deserializers", function() + tab = sample + for _, codec in ipairs({"dump", "bitser"}) do + assert.is_true(Persist.getCodec(codec).id == codec) + ser = Persist.getCodec(codec).serialize + deser = Persist.getCodec(codec).deserialize + str = ser(tab) + assert.are.same(deser(str), tab) + str, ser, deser = nil, nil, nil + end + end) + + it("should work with huge tables", function() + tab = arrayOf(100000) + ser = Persist.getCodec("bitser").serialize + deser = Persist.getCodec("bitser").deserialize + str = ser(tab) + assert.are.same(deser(str), tab) + end) + + it ("should fail to serialize functions", function() + for _, codec in ipairs({"dump", "bitser"}) do + assert.is_true(Persist.getCodec(codec).id == codec) + ser = Persist.getCodec(codec).serialize + deser = Persist.getCodec(codec).deserialize + str = ser(fail) + assert.are_not.same(deser(str), fail) + end + end) + +end)