add table persistence module

base

@@ -1 +1 @@
-Subproject commit dec3df4e5a96d7a039b8e6aeac4000c34c76a83a
+Subproject commit 22d3b504d43e058a0d06c585fda8459749de0c3e

frontend/persist.lua

@@ -0,0 +1,150 @@
+local bitser = require("ffi/bitser")
+local dump = require("dump")
+local lfs = require("libs/libkoreader-lfs")
+local logger = require("logger")
+
+local function readFile(file, bytes)
+    local f, str, err
+    f, err = io.open(file, "rb")
+    if not f then
+        return nil, err
+    end
+    str, err = f:read(bytes or "*a")
+    f:close()
+    if not str then
+        return nil, err
+    end
+    return str
+end
+
+local codecs = {
+    -- bitser: binary form, fast encode/decode, low size. Not human readable.
+    bitser = {
+        id = "bitser",
+        reads_from_file = false,
+
+        serialize = function(t)
+            local ok, str = pcall(bitser.dumps, t)
+            if not ok then
+                return nil, "cannot serialize " .. tostring(t)
+            end
+            return str
+        end,
+
+        deserialize = function(str)
+            local ok, t = pcall(bitser.loads, str)
+            if not ok then
+                return nil, "malformed serialized data"
+            end
+            return t
+        end,
+    },
+    -- dump: human readable, pretty printed, fast enough for most user cases.
+    dump = {
+        id = "dump",
+        reads_from_file = true,
+
+        serialize = function(t, as_bytecode)
+            local content
+            if as_bytecode then
+                local bytecode, err = load("return " .. dump(t))
+                if not bytecode then
+                    logger.warn("cannot convert table to bytecode", err, "fallback to text")
+                else
+                    content = string.dump(bytecode, true)
+                end
+            end
+            if not content then
+                content = "return " .. dump(t)
+            end
+            return content
+        end,
+
+        deserialize = function(str)
+            local t, err = loadfile(str)
+            if not t then
+                t, err = loadstring(str)
+            end
+            if not t then
+                return nil, err
+            end
+            return t()
+        end,
+    }
+}
+
+local Persist = {}
+
+function Persist:new(o)
+    o = o or {}
+    assert(type(o.path) == "string", "path is required")
+    o.codec = o.codec or "dump"
+    setmetatable(o, self)
+    self.__index = self
+    return o
+end
+
+function Persist:exists()
+    local mode = lfs.attributes(self.path, "mode")
+    if mode then
+        return mode == "file"
+    end
+end
+
+function Persist:timestamp()
+    return lfs.attributes(self.path, "modification")
+end
+
+function Persist:size()
+    return lfs.attributes(self.path, "size")
+end
+
+function Persist:load()
+    local t, err
+    if codecs[self.codec].reads_from_file then
+        t, err = codecs[self.codec].deserialize(self.path)
+    else
+        local str
+        str, err = readFile(self.path)
+        if not str then
+            return nil, err
+        end
+        t, err = codecs[self.codec].deserialize(str)
+    end
+    if not t then
+        return nil, err
+    end
+    return t
+end
+
+function Persist:save(t, as_bytecode)
+    local str, file, err
+    str, err = codecs[self.codec].serialize(t, as_bytecode)
+    if not str then
+        return nil, err
+    end
+    file, err = io.open(self.path, "wb")
+    if not file then
+        return nil, err
+    end
+    file:write(str)
+    file:close()
+    return true
+end
+
+function Persist:delete()
+    if not self:exists() then return end
+    return os.remove(self.path)
+end
+
+function Persist.getCodec(name)
+    local fallback = codecs["dump"]
+    for key, codec in pairs(codecs) do
+        if type(key) == "string" and key == name then
+            return codec
+        end
+    end
+    return fallback
+end
+
+return Persist

frontend/util.lua

@@ -1160,23 +1160,6 @@ function util.clearTable(t)
     for i = 0, c do t[i] = nil end
 end
 
---- Dumps a table into a file.
---- @table t the table to be dumped
---- @string file the file to store the table
---- @treturn bool true on success, false otherwise
-function util.dumpTable(t, file)
-    if not t or not file or file == "" then return end
-    local dump = require("dump")
-    local f = io.open(file, "w")
-    if f then
-        f:write("return "..dump(t))
-        f:close()
-        return true
-    end
-    return false
-end
-
-
 --- Encode URL also known as percent-encoding see https://en.wikipedia.org/wiki/Percent-encoding
 --- @string text the string to encode
 --- @treturn encode string

spec/unit/persist_spec.lua

@@ -0,0 +1,89 @@
+describe("Persist module", function()
+    local Persist
+    local sample
+    local bitserInstance, dumpInstance
+    local ser, deser, str, tab
+    local fail = { a = function() end, }
+
+    local function arrayOf(n)
+        assert(type(n) == "number", "wrong type (expected number)")
+        local t = {}
+        for i = 1, n do
+            table.insert(t, i, {
+                a = "sample " .. tostring(i),
+                b = true,
+                c = nil,
+                d = i,
+                e = {
+                    f = {
+                        g = nil,
+                        h = false,
+                    },
+                },
+            })
+        end
+        return t
+    end
+
+    setup(function()
+        require("commonrequire")
+        Persist = require("persist")
+        bitserInstance = Persist:new{ path = "test.dat", codec = "bitser" }
+        dumpInstance = Persist:new { path = "test.txt", codec = "dump" }
+        sample = arrayOf(1000)
+    end)
+
+    it("should save a table to file", function()
+        assert.is_true(bitserInstance:save(sample))
+        assert.is_true(dumpInstance:save(sample))
+    end)
+
+    it("should generate a valid file", function()
+        assert.is_true(bitserInstance:exists())
+        assert.is_true(bitserInstance:size() > 0)
+        assert.is_true(type(bitserInstance:timestamp()) == "number")
+    end)
+
+    it("should load a table from file", function()
+        assert.are.same(sample, bitserInstance:load())
+        assert.are.same(sample, dumpInstance:load())
+    end)
+
+    it("should delete the file", function()
+        bitserInstance:delete()
+        dumpInstance:delete()
+        assert.is_nil(bitserInstance:exists())
+        assert.is_nil(dumpInstance:exists())
+    end)
+
+    it("should return standalone serializers/deserializers", function()
+        tab = sample
+        for _, codec in ipairs({"dump", "bitser"}) do
+            assert.is_true(Persist.getCodec(codec).id == codec)
+            ser = Persist.getCodec(codec).serialize
+            deser = Persist.getCodec(codec).deserialize
+            str = ser(tab)
+            assert.are.same(deser(str), tab)
+            str, ser, deser = nil, nil, nil
+        end
+    end)
+
+    it("should work with huge tables", function()
+        tab = arrayOf(100000)
+        ser = Persist.getCodec("bitser").serialize
+        deser = Persist.getCodec("bitser").deserialize
+        str = ser(tab)
+        assert.are.same(deser(str), tab)
+    end)
+
+    it ("should fail to serialize functions", function()
+        for _, codec in ipairs({"dump", "bitser"}) do
+            assert.is_true(Persist.getCodec(codec).id == codec)
+            ser = Persist.getCodec(codec).serialize
+            deser = Persist.getCodec(codec).deserialize
+            str = ser(fail)
+            assert.are_not.same(deser(str), fail)
+        end
+    end)
+
+end)