From bc0a55f093dae3974e0576e866a041bf23597ee2 Mon Sep 17 00:00:00 2001 From: Utsob Roy Date: Sat, 7 May 2022 02:44:28 +0600 Subject: [PATCH] Refactor exporter.koplugin (#8944) Changed: - select multiple targets and export to them in a single click. - local targets (html, json and text) now are timestamped. Exporting booknotes on already exported documents will generate a new file with all the highlights present at export time. Previous files won't be deleted. Fixed: - chapters are now correctly represented in html output. - json issues when exporting the whole history. - joplin and readwise crashes when they're unable to reach the server - joplin update notes mechanism. - joplin is able to recreate the notebook if the user deletes or renames its current one. - highlights of read-only documents are also added when exporting the whole history (affects mostly android, might affect desktop targets) Co-authored-by: Utsob Roy --- plugins/exporter.koplugin/JoplinClient.lua | 154 ---- plugins/exporter.koplugin/ReadwiseClient.lua | 70 -- plugins/exporter.koplugin/base.lua | 135 +++ plugins/exporter.koplugin/clip.lua | 3 +- plugins/exporter.koplugin/main.lua | 805 ++++-------------- plugins/exporter.koplugin/target/html.lua | 62 ++ plugins/exporter.koplugin/target/joplin.lua | 364 ++++++++ plugins/exporter.koplugin/target/json.lua | 49 ++ plugins/exporter.koplugin/target/readwise.lua | 135 +++ plugins/exporter.koplugin/target/text.lua | 46 + .../exporter.koplugin/{ => template}/note.tpl | 8 +- .../exporter.koplugin/{ => template}/slt2.lua | 0 spec/unit/exporter_plugin_main_spec.lua | 63 +- 13 files changed, 980 insertions(+), 914 deletions(-) delete mode 100644 plugins/exporter.koplugin/JoplinClient.lua delete mode 100644 plugins/exporter.koplugin/ReadwiseClient.lua create mode 100644 plugins/exporter.koplugin/base.lua create mode 100644 plugins/exporter.koplugin/target/html.lua create mode 100644 plugins/exporter.koplugin/target/joplin.lua create mode 100644 plugins/exporter.koplugin/target/json.lua create mode 100644 plugins/exporter.koplugin/target/readwise.lua create mode 100644 plugins/exporter.koplugin/target/text.lua rename plugins/exporter.koplugin/{ => template}/note.tpl (92%) rename plugins/exporter.koplugin/{ => template}/slt2.lua (100%) diff --git a/plugins/exporter.koplugin/JoplinClient.lua b/plugins/exporter.koplugin/JoplinClient.lua deleted file mode 100644 index 3048e054f..000000000 --- a/plugins/exporter.koplugin/JoplinClient.lua +++ /dev/null @@ -1,154 +0,0 @@ -local http = require("socket.http") -local json = require("json") -local ltn12 = require("ltn12") -local socketutil = require("socketutil") - -local JoplinClient = { - server_ip = "localhost", - server_port = 41184, - auth_token = "" -} - -function JoplinClient:new(o) - o = o or {} - self.__index = self - setmetatable(o, self) - return o -end - -function JoplinClient:_makeRequest(url, method, request_body) - local sink = {} - local request_body_json = json.encode(request_body) - local source = ltn12.source.string(request_body_json) - socketutil:set_timeout(socketutil.LARGE_BLOCK_TIMEOUT, socketutil.LARGE_TOTAL_TIMEOUT) - http.request{ - url = url, - method = method, - sink = ltn12.sink.table(sink), - source = source, - headers = { - ["Content-Length"] = #request_body_json, - ["Content-Type"] = "application/json" - }, - } - socketutil:reset_timeout() - - if not sink[1] then - error("No response from Joplin Server") - end - - local response = json.decode(sink[1]) - - if response.error then - error(response.error) - end - - return response -end - -function JoplinClient:ping() - local sink = {} - - http.request{ - url = "http://"..self.server_ip..":"..self.server_port.."/ping", - method = "GET", - sink = ltn12.sink.table(sink) - } - - if sink[1] == "JoplinClipperServer" then - return true - else - return false - end -end - --- If successful returns id of found note. -function JoplinClient:findNoteByTitle(title, notebook_id) - local url_base = "http://"..self.server_ip..":"..self.server_port.."/notes?".."token="..self.auth_token.."&fields=id,title,parent_id&page=" - - local url - local page = 1 - local has_more - - repeat - url = url_base..page - local notes = self:_makeRequest(url, "GET") - has_more = notes.has_more - for _, note in ipairs(notes.items) do - if note.title == title then - if notebook_id == nil or note.parent_id == notebook_id then - return note.id - end - end - end - page = page + 1 - until not has_more - return false - -end - --- If successful returns id of found notebook (folder). -function JoplinClient:findNotebookByTitle(title) - local url_base = "http://"..self.server_ip..":"..self.server_port.."/folders?".."token="..self.auth_token.."&".."query="..title.."&page=" - - - local url - local page = 1 - local has_more - - repeat - url = url_base..page - local folders = self:_makeRequest(url, "GET") - has_more = folders.has_more - for _, folder in ipairs(folders.items) do - if folder.title == title then - return folder.id - end - end - page = page + 1 - until not has_more - return false -end - --- If successful returns id of created notebook (folder). -function JoplinClient:createNotebook(title, created_time) - local request_body = { - title = title, - created_time = created_time - } - - local url = "http://"..self.server_ip..":"..self.server_port.."/folders?".."token="..self.auth_token - local response = self:_makeRequest(url, "POST", request_body) - - return response.id -end - - --- If successful returns id of created note. -function JoplinClient:createNote(title, note, parent_id, created_time) - local request_body = { - title = title, - body = note, - parent_id = parent_id, - created_time = created_time - } - local url = "http://"..self.server_ip..":"..self.server_port.."/notes?".."token="..self.auth_token - local response = self:_makeRequest(url, "POST", request_body) - - return response.id -end - --- If successful returns id of updated note. -function JoplinClient:updateNote(note_id, note, title, parent_id) - local request_body = { - body = note, - title = title, - parent_id = parent_id - } - - local url = "http://"..self.server_ip..":"..self.server_port.."/notes/"..note_id.."?token="..self.auth_token - local response = self:_makeRequest(url, "PUT", request_body) - return response.id -end - -return JoplinClient diff --git a/plugins/exporter.koplugin/ReadwiseClient.lua b/plugins/exporter.koplugin/ReadwiseClient.lua deleted file mode 100644 index 61d8e774e..000000000 --- a/plugins/exporter.koplugin/ReadwiseClient.lua +++ /dev/null @@ -1,70 +0,0 @@ -local http = require("socket.http") -local json = require("json") -local logger = require("logger") -local ltn12 = require("ltn12") -local socket = require("socket") -local socketutil = require("socketutil") - -local ReadwiseClient = { - auth_token = "" -} - -function ReadwiseClient:new(o) - o = o or {} - self.__index = self - setmetatable(o, self) - return o -end - -function ReadwiseClient:_makeRequest(endpoint, method, request_body) - local sink = {} - local request_body_json = json.encode(request_body) - local source = ltn12.source.string(request_body_json) - socketutil:set_timeout(socketutil.LARGE_BLOCK_TIMEOUT, socketutil.LARGE_TOTAL_TIMEOUT) - local request = { - url = "https://readwise.io/api/v2/" .. endpoint, - method = method, - sink = ltn12.sink.table(sink), - source = source, - headers = { - ["Content-Length"] = #request_body_json, - ["Content-Type"] = "application/json", - ["Authorization"] = "Token " .. self.auth_token - }, - } - local code, _, status = socket.skip(1, http.request(request)) - socketutil:reset_timeout() - - if code ~= 200 then - logger.warn("ReadwiseClient: HTTP response code <> 200. Response status: ", status) - error("ReadwiseClient: HTTP response code <> 200.") - end - - local response = json.decode(sink[1]) - - return response -end - -function ReadwiseClient:createHighlights(booknotes) - local highlights = {} - for _, chapter in ipairs(booknotes) do - for _, clipping in ipairs(chapter) do - local highlight = { - text = clipping.text, - title = booknotes.title, - author = booknotes.author ~= "" and booknotes.author or nil, -- optional author - source_type = "koreader", - category = "books", - note = clipping.note, - location = clipping.page, - location_type = "page", - highlighted_at = os.date("!%Y-%m-%dT%TZ", clipping.time), - } - table.insert(highlights, highlight) - end - end - local result = self:_makeRequest("highlights", "POST", { highlights = highlights }) - logger.dbg("ReadwiseClient createHighlights result", result) -end - -return ReadwiseClient diff --git a/plugins/exporter.koplugin/base.lua b/plugins/exporter.koplugin/base.lua new file mode 100644 index 000000000..3fd5697c5 --- /dev/null +++ b/plugins/exporter.koplugin/base.lua @@ -0,0 +1,135 @@ +--[[-- +Base for highlight exporters. + +Each target should inherit from this class and implement *at least* an `export` function. + +@module baseexporter +]] + +local BaseExporter = { + clipping_dir = require("datastorage"):getDataDir() .. "/clipboard" +} + +function BaseExporter:new(o) + o = o or {} + assert(type(o.name) == "string", "name is mandatory") + setmetatable(o, self) + self.__index = self + return o:_init() +end + +function BaseExporter:_init() + self.extension = self.extension or self.name + self.is_remote = self.is_remote or false + self.version = self.version or "1.0.0" + self:loadSettings() + return self +end + +--[[-- +Export timestamp + +@treturn string timestamp +]] +function BaseExporter:getTimeStamp() + local ts = self.timestamp or os.time() + return os.date("%Y-%m-%d %H:%M:%S", ts) +end + +--[[-- +Exporter version + +@treturn string version +]] +function BaseExporter:getVersion() + return self.name .. "/" .. self.version +end + +--[[-- +Loads settings for the exporter +]] +function BaseExporter:loadSettings() + local plugin_settings = G_reader_settings:readSetting("exporter") or {} + self.settings = plugin_settings[self.name] or {} +end + +--[[-- +Saves settings for the exporter +]] +function BaseExporter:saveSettings() + local plugin_settings = G_reader_settings:readSetting("exporter") or {} + plugin_settings[self.name] = self.settings + G_reader_settings:saveSetting("exporter", plugin_settings) + self.new_settings = true +end + +--[[-- +Exports a table of booknotes to local format or remote service + +@param t table of booknotes +@treturn bool success +]] +function BaseExporter:export(t) end + +--[[-- +File path where the exporter writes its output + +@param t table of booknotes +@treturn string absolute path or nil +]] +function BaseExporter:getFilePath(t) + if not self.is_remote then + return string.format("%s/%s-%s.%s", + self.clipping_dir, + self:getTimeStamp(), + #t == 1 and t[1].title or "all-books", + self.extension) + end +end + +--[[-- +Configuration menu for the exporter + +@treturn table menu with exporter settings +]] +function BaseExporter:getMenuTable() + return { + text = self.name:gsub("^%l", string.upper), + checked_func = function() + return self:isEnabled() + end, + callback = function() + self:toggleEnabled() + end, + } +end + +--[[-- +Checks if the exporter is ready to export + +@treturn bool ready +]] +function BaseExporter:isReadyToExport() + return true +end + +--[[-- +Checks if the exporter was enabled by the user and it is ready to export + +@treturn bool enabled +]] +function BaseExporter:isEnabled() + return self.settings.enabled and self:isReadyToExport() +end + +--[[-- +Toggles exporter enabled state if it's ready to export +]] +function BaseExporter:toggleEnabled() + if self:isReadyToExport() then + self.settings.enabled = not self.settings.enabled + self:saveSettings() + end +end + +return BaseExporter diff --git a/plugins/exporter.koplugin/clip.lua b/plugins/exporter.koplugin/clip.lua index 69ef11ca3..99c830868 100644 --- a/plugins/exporter.koplugin/clip.lua +++ b/plugins/exporter.koplugin/clip.lua @@ -1,4 +1,3 @@ -local DataStorage = require("datastorage") local DocumentRegistry = require("document/documentregistry") local DocSettings = require("docsettings") local ReadHistory = require("readhistory") @@ -10,7 +9,7 @@ local T = require("ffi/util").template local MyClipping = { my_clippings = "/mnt/us/documents/My Clippings.txt", - history_dir = DataStorage:getDataDir() .. "/history", + history_dir = "./history", } function MyClipping:new(o) diff --git a/plugins/exporter.koplugin/main.lua b/plugins/exporter.koplugin/main.lua index 02cff0502..94b08b583 100644 --- a/plugins/exporter.koplugin/main.lua +++ b/plugins/exporter.koplugin/main.lua @@ -1,435 +1,67 @@ -local BD = require("ui/bidi") -local InputContainer = require("ui/widget/container/inputcontainer") -local InfoMessage = require("ui/widget/infomessage") -local NetworkMgr = require("ui/network/manager") -local DataStorage = require("datastorage") -local DocSettings = require("docsettings") -local InputDialog = require("ui/widget/inputdialog") -local UIManager = require("ui/uimanager") -local logger = require("logger") -local util = require("ffi/util") -local Device = require("device") -local JoplinClient = require("JoplinClient") -local ReadwiseClient = require("ReadwiseClient") -local T = util.template -local _ = require("gettext") -local N_ = _.ngettext -local slt2 = require('slt2') -local MyClipping = require("clip") -local json = require("json") - -local function getOrMigrateSettings() - local settings = G_reader_settings:readSetting("exporter") - if not settings then - -- migrate settings from old plugin and remove specific evernote ones. - settings = G_reader_settings:readSetting("evernote") - if type(settings) == "table" then - settings.domain = nil - settings.username = nil - settings.token = nil - end - end - return settings or {} -end - -local Exporter = InputContainer:new{ - name = "exporter", - notebook_name = _("KOReader Notes"), - notemarks = _("Note: "), - clipping_dir = DataStorage:getDataDir() .. "/clipboard", - - evernote_token = nil, - notebook_guid = nil, -} - -function Exporter:init() - self.text_clipping_file = self.clipping_dir .. "/KOReaderClipping.txt" - self.json_clipping_file = self.clipping_dir .. "/KOReaderClipping.json" - local settings = getOrMigrateSettings() - self.notebook_guid = settings.notebook - self.joplin_IP = settings.joplin_IP or "localhost" - self.joplin_port = settings.joplin_port or 41185 - self.joplin_token = settings.joplin_token -- or your token - self.joplin_notebook_guid = settings.joplin_notebook_guid or nil - self.readwise_token = settings.readwise_token or nil - self.html_export = settings.html_export or false - self.joplin_export = settings.joplin_export or false - self.txt_export = settings.txt_export or false - self.json_export = settings.json_export or false - self.readwise_export = settings.readwise_export or false - --- @todo Is this if block necessary? Nowhere in the code they are assigned both true. - -- Do they check against external modifications to settings file? +--[[ + Export highlights to different targets. - if self.html_export then - self.txt_export = false - self.joplin_export = false - self.json_export = false - self.readwise_export = false - elseif self.txt_export then - self.joplin_export = false - self.json_export = false - self.readwise_export = false - elseif self.json_export then - self.joplin_export = false - self.readwise_export = false - end + Some conventions: - self.parser = MyClipping:new{ - my_clippings = "/mnt/us/documents/My Clippings.txt", - history_dir = "./history", - } - self.template = slt2.loadfile(self.path.."/note.tpl") - self:migrateClippings() - self.config = DocSettings:open(util.joinPath(self.clipping_dir, "exporter.sdr")) + - Target: each local format or remote service this plugin can translate to. - self.ui.menu:registerToMainMenu(self) -end + Each new target should inherit from "formats/base" and implement *at least* an export function. -function Exporter:isDocless() - return self.ui == nil or self.ui.document == nil or self.view == nil -end + - Highlight: Text or image in document. Stored in "highlights" table of documents sidecar file. -function Exporter:readyToExport() - return self.html_export ~= false or - self.txt_export ~= false or - self.json_export ~= false or - self.joplin_export ~= false or - self.readwise_export ~= false -end + Parser uses this table. + If highlight._._.text field is empty the parser uses highlight._._.pboxes field to get an image instead. -function Exporter:migrateClippings() - if jit.os == "OSX" then return end - local old_dir = util.joinPath(util.realpath(util.joinPath(self.path, "..")), - "evernote.sdr") - if lfs.attributes(old_dir, "mode") == "directory" then - local mv_bin = Device:isAndroid() and "/system/bin/mv" or "/bin/mv" - return util.execute(mv_bin, old_dir, self.clipping_dir) == 0 - end -end + - Bookmarks: Data in bookmark explorer. Stored in "bookmarks" table of documents sidecar file. -function Exporter:addToMainMenu(menu_items) - menu_items.exporter = { - text = _("Export highlights"), - sub_item_table = { - { - text = _("Joplin") , - checked_func = function() return self.joplin_export end, - sub_item_table ={ - { - text = _("Set Joplin IP and Port"), - keep_menu_open = true, - callback = function() - local MultiInputDialog = require("ui/widget/multiinputdialog") - local url_dialog - url_dialog = MultiInputDialog:new{ - title = _("Set Joplin IP and port number"), - fields = { - { - text = self.joplin_IP, - input_type = "string" - }, - { - text = self.joplin_port, - input_type = "number" - } - }, - buttons = { - { - { - text = _("Cancel"), - id = "close", - callback = function() - UIManager:close(url_dialog) - end - }, - { - text = _("OK"), - callback = function() - local fields = url_dialog:getFields() - local ip = fields[1] - local port = tonumber(fields[2]) - if ip ~= "" then - if port and port < 65355 then - self.joplin_IP = ip - self.joplin_port = port - end - self:saveSettings() - end - UIManager:close(url_dialog) - end - } - } - } - } - UIManager:show(url_dialog) - url_dialog:onShowKeyboard() - end - }, - { - text = _("Set authorization token"), - keep_menu_open = true, - callback = function() - local auth_dialog - auth_dialog = InputDialog:new{ - title = _("Set authorization token for Joplin"), - input = self.joplin_token, - buttons = { - { - { - text = _("Cancel"), - id = "close", - callback = function() - UIManager:close(auth_dialog) - end - }, - { - text = _("Set token"), - callback = function() - self.joplin_token = auth_dialog:getInputText() - self:saveSettings() - UIManager:close(auth_dialog) - end - } - } - } - } - UIManager:show(auth_dialog) - auth_dialog:onShowKeyboard() - end - }, - { - text = _("Export to Joplin"), - checked_func = function() return self.joplin_export end, - callback = function() - self.joplin_export = not self.joplin_export - if self.joplin_export then - self.html_export = false - self.txt_export = false - self.json_export = false - self.readwise_export = false - end - self:saveSettings() - end - }, - { - text = _("Help"), - keep_menu_open = true, - callback = function() - UIManager:show(InfoMessage:new{ - text = T(_([[You can enter your auth token on your computer by saving an empty token. Then quit KOReader, edit the exporter.joplin_token field in %1/settings.reader.lua after creating a backup, and restart KOReader once you're done. - -To export to Joplin, you must forward the IP and port used by this plugin to the localhost:port on which Joplin is listening. This can be done with socat or a similar program. For example: - -For Windows: netsh interface portproxy add v4tov4 listenaddress=0.0.0.0 listenport=41185 connectaddress=localhost connectport=41184 + Every field in bookmarks._ has "text" and "notes" fields. + When user edits a highlight or "renames" bookmark the text field is created or updated. + The parser looks to bookmarks._.text field for edited notes. bookmarks._.notes isn't used for exporting operations. -For Linux: $socat tcp-listen:41185,reuseaddr,fork tcp:localhost:41184 + - Clippings: Parsed form of highlights. Single table for all documents. -For more information, please visit https://github.com/koreader/koreader/wiki/Highlight-export.]]) - , BD.dirpath(DataStorage:getDataDir())) - }) - end - } - } - }, - { - text = _("Readwise") , - checked_func = function() return self.readwise_export end, - separator = true, - sub_item_table ={ - { - text = _("Set authorization token"), - keep_menu_open = true, - callback = function() - local auth_dialog - auth_dialog = InputDialog:new{ - title = _("Set authorization token for Readwise"), - input = self.readwise_token, - buttons = { - { - { - text = _("Cancel"), - id = "close", - callback = function() - UIManager:close(auth_dialog) - end - }, - { - text = _("Set token"), - callback = function() - self.readwise_token = auth_dialog:getInputText() - self:saveSettings() - UIManager:close(auth_dialog) - end - } - } - } - } - UIManager:show(auth_dialog) - auth_dialog:onShowKeyboard() - end - }, - { - text = _("Export to Readwise"), - checked_func = function() return self.readwise_export end, - callback = function() - self.readwise_export = not self.readwise_export - if self.readwise_export then - self.html_export = false - self.txt_export = false - self.json_export = false - self.joplin_export = false - end - self:saveSettings() - end - }, - { - text = _("Help"), - keep_menu_open = true, - callback = function() - UIManager:show(InfoMessage:new{ - text = T(_([[You can enter your auth token on your computer by saving an empty token. Then quit KOReader, edit the exporter.readwise_token field in %1/settings.reader.lua after creating a backup, and restart KOReader once you're done. - -For more information, please visit https://github.com/koreader/koreader/wiki/Highlight-export.]]) - , BD.dirpath(DataStorage:getDataDir())) - }) - end - } - } - }, - { - text = _("Export all notes in this book"), - enabled_func = function() - return not self:isDocless() and self:readyToExport() and not self.txt_export - end, - callback = function() - local export_callback = function() - UIManager:nextTick(function() - self:exportCurrentNotes(self.view) - end) + - Booknotes: Every table in clippings table. clippings = {"title" = booknotes} - UIManager:show(InfoMessage:new{ - text = _("Exporting may take several seconds…"), - timeout = 1, - }) - end - if self.joplin_export or self.readwise_export then - NetworkMgr:runWhenOnline(export_callback) - else - export_callback() - end - end - }, - { - text = _("Export all notes in your library"), - enabled_func = function() - return self:readyToExport() - end, - callback = function() - local export_callback = function() - UIManager:nextTick(function() - self:exportAllNotes() - end) +--]] - UIManager:show(InfoMessage:new{ - text = _("Exporting may take several minutes…"), - timeout = 1, - }) - end - if self.joplin_export or self.readwise_export then - NetworkMgr:runWhenOnline(export_callback) - else - export_callback() - end +local DataStorage = require("datastorage") +local Device = require("device") +local InfoMessage = require("ui/widget/infomessage") +local InputContainer = require("ui/widget/container/inputcontainer") +local MyClipping = require("clip") +local NetworkMgr = require("ui/network/manager") +local UIManager = require("ui/uimanager") +local logger = require("logger") +local _ = require("gettext") - end, - separator = true, - }, - { - text = _("Export to local JSON files"), - checked_func = function() return self.json_export end, - callback = function() - self.json_export = not self.json_export - if self.json_export then - self.txt_export = false - self.html_export = false - self.joplin_export = false - self.readwise_export = false - end - self:saveSettings() - end - }, - { - text = _("Export to local HTML files"), - checked_func = function() return self.html_export end, - callback = function() - self.html_export = not self.html_export - if self.html_export then - self.txt_export = false - self.json_export = false - self.joplin_export = false - self.readwise_export = false - end - self:saveSettings() - end - }, - { - text = _("Export to local clipping text file"), - checked_func = function() return self.txt_export end, - callback = function() - self.txt_export = not self.txt_export - if self.txt_export then - self.html_export = false - self.json_export = false - self.joplin_export = false - self.readwise_export = false - end - self:saveSettings() - end, - separator = true, - }, - { - text = _("Purge history records"), - callback = function() - self.config:purge() - UIManager:show(InfoMessage:new{ - text = _("History records have been purged.\nAll notes will be exported again next time.\n"), - timeout = 2, - }) - end - } - } - } -end -function Exporter:saveSettings() - local settings = { - notebook = self.notebook_guid, - html_export = self.html_export, - txt_export = self.txt_export, - json_export = self.json_export, - joplin_IP = self.joplin_IP, - joplin_port = self.joplin_port, - joplin_token = self.joplin_token, - joplin_notebook_guid = self.joplin_notebook_guid, - joplin_export = self.joplin_export, - readwise_token = self.readwise_token, - readwise_export = self.readwise_export - } - G_reader_settings:saveSetting("exporter", settings) -end +-- migrate settings from old "evernote.koplugin" or from previous (monolithic) "exporter.koplugin" +local function migrateSettings() + local formats = { "html", "joplin", "json", "readwise", "text" } -function Exporter:getExportNotebook(client) - local name = self.notebook_name - return client:findNotebookByTitle(name) or client:createNotebook(name).guid -end + local settings = G_reader_settings:readSetting("exporter") + if not settings then + settings = G_reader_settings:readSetting("evernote") + end -function Exporter:exportCurrentNotes(view) - local clippings = self.parser:parseCurrentDoc(view) - self:exportClippings(clippings) + if type(settings) == "table" then + for _, fmt in ipairs(formats) do + if type(settings[fmt]) == "table" then return end + end + local new_settings = {} + for _, fmt in ipairs(formats) do + new_settings[fmt] = { enabled = false } + end + new_settings["joplin"].ip = settings.joplin_IP + new_settings["joplin"].port = settings.joplin_port + new_settings["joplin"].token = settings.joplin_token + new_settings["readwise"].token = settings.readwise_token + G_reader_settings:saveSetting("exporter", new_settings) + end end -function Exporter:updateHistoryClippings(clippings, new_clippings) - -- update clippings from history clippings +-- update clippings from history clippings +local function updateHistoryClippings(clippings, new_clippings) for title, booknotes in pairs(new_clippings) do for chapter_index, chapternotes in ipairs(booknotes) do for note_index, note in ipairs(chapternotes) do @@ -448,7 +80,8 @@ function Exporter:updateHistoryClippings(clippings, new_clippings) return clippings end -function Exporter:updateMyClippings(clippings, new_clippings) +-- update clippings from Kindle annotation system +local function updateMyClippings(clippings, new_clippings) -- only new titles or new notes in My clippings are updated to clippings -- since appending is the only way to modify notes in My Clippings for title, booknotes in pairs(new_clippings) do @@ -460,265 +93,141 @@ function Exporter:updateMyClippings(clippings, new_clippings) return clippings end ---[[-- -Parses highlights and calls exporter functions. +local Exporter = InputContainer:new { + name = "exporter", + clipping_dir = DataStorage:getDataDir() .. "/clipboard", + targets = { + html = require("target/html"), + joplin = require("target/joplin"), + json = require("target/json"), + readwise = require("target/readwise"), + text = require("target/text"), + }, +} -Entry point for exporting highlights. User interface calls this function. -Parses current document and documents from history, passes them to exportClippings(). -Highlight: Highlighted text or image in document, stored in "highlights" table in -documents sidecar file. Parser uses this table. If highlight._._.text field is empty parser uses -highlight._._.pboxes field to get an image instead. -Bookmarks: Data in bookmark explorer. Stored in "bookmarks" table of documents sidecar file. Every -field in bookmarks._ has "text" and "notes" fields When user edits a highlight or "renames" bookmark, -text field is created or updated. Parser looks to bookmarks._.text field for edited notes. bookmarks._.notes isn't used for exporting operations. -https://github.com/koreader/koreader/blob/605f6026bbf37856ee54741b8a0697337ca50039/plugins/evernote.koplugin/clip.lua#L229 -Clippings: Parsed form of highlights, stored in clipboard/evernote.sdr/metadata.sdr.lua -for all documents. Used only for exporting bookmarks. Internal highlight or bookmark functions -does not use this table. -Booknotes: Every table in clippings table. clippings = {"title" = booknotes} ---]] -function Exporter:exportAllNotes() - -- Flush highlights of current document. - if not self:isDocless() then - self.ui:saveSettings() - end - local clippings = self.config:readSetting("clippings") or {} - clippings = self:updateHistoryClippings(clippings, self.parser:parseHistory()) - clippings = self:updateMyClippings(clippings, self.parser:parseMyClippings()) - -- remove blank entries - for title, booknotes in pairs(clippings) do - -- chapter number is zero - if #booknotes == 0 then - clippings[title] = nil - end +function Exporter:init() + migrateSettings() + self.parser = MyClipping:new { + history_dir = DataStorage:getDataDir() .. "/history", + } + for k, _ in pairs(self.targets) do + self.targets[k].path = self.path end - --logger.dbg("clippings", clippings) - self:exportClippings(clippings) - self.config:saveSetting("clippings", clippings) - self.config:flush() + self.ui.menu:registerToMainMenu(self) end -function Exporter:exportClippings(clippings) - local exported_stamp - local joplin_client - local readwise_client - if self.html_export then - exported_stamp= "html" - elseif self.json_export then - exported_stamp= "json" - elseif self.txt_export then - os.remove(self.text_clipping_file) - exported_stamp = "txt" - elseif self.joplin_export then - exported_stamp = "joplin" - joplin_client = JoplinClient:new{ - server_ip = self.joplin_IP, - server_port = self.joplin_port, - auth_token = self.joplin_token - } - ---@todo Check if user deleted our notebook, in that case note - -- will end up in random folder in Joplin. - if not self.joplin_notebook_guid then - self.joplin_notebook_guid = joplin_client:createNotebook(self.notebook_name) - self:saveSettings() - end - elseif self.readwise_export then - exported_stamp = "readwise" - readwise_client = ReadwiseClient:new{ - auth_token = self.readwise_token - } - else - assert("an exported_stamp is expected for a new export type") - end - - local export_count, error_count = 0, 0 - local export_title, error_title - for title, booknotes in pairs(clippings) do - if type(booknotes.exported) ~= "table" then - booknotes.exported = {} +function Exporter:isReady() + for k, v in pairs(self.targets) do + if v:isEnabled() then + return true end - -- check if booknotes are exported in this notebook - -- so that booknotes will still be exported after switching user account - -- Don't respect exported_stamp on txt export since it isn't possible to delete(update) prior clippings. - if booknotes.exported[exported_stamp] ~= true or self.txt_export or self.json_export then - local ok, err - if self.html_export then - ok, err = pcall(self.exportBooknotesToHTML, self, title, booknotes) - elseif self.txt_export then - ok, err = pcall(self.exportBooknotesToTXT, self, title, booknotes) - elseif self.json_export then - ok, err = pcall(self.exportBooknotesToJSON, self, title, booknotes) - elseif self.joplin_export then - ok, err = pcall(self.exportBooknotesToJoplin, self, joplin_client, title, booknotes) - elseif self.readwise_export then - ok, err = pcall(self.exportBooknotesToReadwise, self, readwise_client, title, booknotes) - end - -- Error reporting - if not ok and err and err:find("Transport not open") then - --- @note: No recursive callback because it feels fishy here... - NetworkMgr:beforeWifiAction() - return - elseif not ok and err then - logger.dbg("Error while exporting book", title, err) - error_count = error_count + 1 - error_title = title - elseif ok then - logger.dbg("Exported notes in book:", title) - export_count = export_count + 1 - export_title = title - booknotes.exported[exported_stamp] = true - end - end - end - - local msg = "Nothing was exported." - local all_count = export_count + error_count - if export_count > 0 and error_count == 0 then - msg = T( - N_("Exported notes from the book:\n%1", - "Exported notes from the book:\n%1\nand %2 others.", - all_count-1), - export_title, - all_count-1 - ) - elseif error_count > 0 then - msg = T( - N_("An error occurred while trying to export notes from the book:\n%1", - "Multiple errors occurred while trying to export notes from the book:\n%1\nand %2 others.", - error_count-1), - error_title, - error_count-1 - ) - end - if (self.html_export or self.txt_export) and export_count > 0 then - msg = msg .. T(_("\nNotes can be found in %1/."), BD.dirpath(util.realpath(self.clipping_dir))) end - UIManager:show(InfoMessage:new{ text = msg }) + return false end -function Exporter:exportBooknotesToHTML(title, booknotes) - local content = slt2.render(self.template, { - booknotes = booknotes, - notemarks = self.notemarks, - }) - --logger.dbg("content", content) - local html = io.open(self.clipping_dir .. "/" .. title .. ".html", "w") - if html then - html:write(content) - html:close() - end -end - -function Exporter:prepareBooknotesForJSON(booknotes) - local exportable = { - title = booknotes.title, - author = booknotes.author, - entries = {}, - exported = booknotes.exported, - file = booknotes.file - } - for _, entry in ipairs(booknotes) do - table.insert(exportable.entries, entry[1]) - end - return exportable +function Exporter:isDocReady() + local docless = self.ui == nil or self.ui.document == nil or self.view == nil + return not docless and self:isReady() end --- This function should handle both multidocument export and single exports. --- For Single Exports, it will create a JSON file with a object ({}) as root node. --- For Multidocument export, it will create a JSON file with an array ([]) as root node. -function Exporter:exportToJSON(clippings) - local file = io.open(self.json_clipping_file, "a") - if file then - local exportable = {} - if table.getn(clippings) == 1 then - -- We will handle single document export here. - exportable = self:prepareBooknotesForJSON(clippings[0]) - else - for _, booknotes in ipairs(clippings) do - table.insert(exportable, self:prepareBooknotesForJSON(booknotes)) +function Exporter:requiresNetwork() + for k, v in pairs(self.targets) do + if v:isEnabled() then + if v.is_remote then + return true end end - file:write(json.encode(exportable)) - file:write("\n") - file:close() end end -function Exporter:exportBooknotesToJSON(title, booknotes) - logger.dbg("booknotes", booknotes) - local file = io.open(self.json_clipping_file, "a") - if file then - local exportable = self:prepareBooknotesForJSON(booknotes) - file:write(json.encode(exportable)) - file:write("\n") - file:close() - end +function Exporter:exportCurrentNotes() + local clippings = self.parser:parseCurrentDoc(self.view) + self:exportClippings(clippings) end -function Exporter:exportBooknotesToTXT(title, booknotes) - -- Use wide_space to avoid crengine to treat it specially. - local wide_space = "\227\128\128" - local file = io.open(self.text_clipping_file, "a") - if file then - file:write(title .. "\n" .. wide_space .. "\n") - for _ignore1, chapter in ipairs(booknotes) do - if chapter.title then - file:write(wide_space .. chapter.title .. "\n" .. wide_space .. "\n") - end - for _ignore2, clipping in ipairs(chapter) do - file:write(wide_space .. wide_space .. - T(_("-- Page: %1, added on %2\n"), - clipping.page, os.date("%c", clipping.time))) - if clipping.text then - file:write(clipping.text) - end - if clipping.note then - file:write("\n---\n" .. clipping.note) - end - if clipping.image then - file:write(_("")) - end - file:write("\n-=-=-=-=-=-\n") - end +function Exporter:exportAllNotes() + local clippings = {} + clippings = updateHistoryClippings(clippings, self.parser:parseHistory()) + if Device:isKindle() then + clippings = updateMyClippings(clippings, self.parser:parseMyClippings()) + end + for title, booknotes in pairs(clippings) do + -- chapter number is zero + if #booknotes == 0 then + clippings[title] = nil end - - file:write("\n") - file:close() end + self:exportClippings(clippings) end -function Exporter:exportBooknotesToJoplin(client, title, booknotes) - if not client:ping() then - error("Cannot reach Joplin server") - end - - local note_guid = client:findNoteByTitle(title, self.joplin_notebook_guid) - local note = "" - for _, chapter in ipairs(booknotes) do - if chapter.title then - note = note .. "\n\t*" .. chapter.title .. "*\n\n * * *" - end - - for _, clipping in ipairs(chapter) do - note = note .. os.date("%Y-%m-%d %H:%M:%S \n", clipping.time) - note = note .. clipping.text - if clipping.note then - note = note .. "\n---\n" .. clipping.note +function Exporter:exportClippings(clippings) + if type(clippings) ~= "table" then return end + local exportables = {} + for _title, booknotes in pairs(clippings) do + table.insert(exportables, booknotes) + end + local export_callback = function() + UIManager:nextTick(function() + local timestamp = os.time() + for k, v in pairs(self.targets) do + if v:isEnabled() then + v.timestamp = timestamp + v:export(exportables) + v.timestamp = nil + end end - note = note .. "\n * * *\n" - end - end + end) - if note_guid then - client:updateNote(note_guid, note) + UIManager:show(InfoMessage:new { + text = _("Exporting may take several seconds…"), + timeout = 1, + }) + end + if self:requiresNetwork() then + NetworkMgr:runWhenOnline(export_callback) else - client:createNote(title, note, self.joplin_notebook_guid) + export_callback() end - end -function Exporter:exportBooknotesToReadwise(client, title, booknotes) - client:createHighlights(booknotes) +function Exporter:addToMainMenu(menu_items) + local submenu = {} + for k, v in pairs(self.targets) do + submenu[#submenu + 1] = v:getMenuTable() + end + table.sort(submenu, function(v1, v2) + return v1.text < v2.text + end) + + menu_items.exporter = { + text = _("Export highlights"), + sub_item_table = { + { + text = _("Export all notes in this book"), + enabled_func = function() + return self:isDocReady() + end, + callback = function() + self:exportCurrentNotes() + end, + }, + { + text = _("Export all notes in your library"), + enabled_func = function() + return self:isReady() + end, + callback = function() + self:exportAllNotes() + end, + separator = true, + }, + { + text = _("Choose formats and services"), + sub_item_table = submenu, + separator = true, + }, + } + } end return Exporter diff --git a/plugins/exporter.koplugin/target/html.lua b/plugins/exporter.koplugin/target/html.lua new file mode 100644 index 000000000..7f003a8a7 --- /dev/null +++ b/plugins/exporter.koplugin/target/html.lua @@ -0,0 +1,62 @@ +local logger = require("logger") +local slt2 = require("template/slt2") + +-- html exporter +local HtmlExporter = require("base"):new { + name = "html", +} + +local function format(booknotes) + local chapters = {} + local curr_chapter = nil + for _, booknote in ipairs(booknotes) do + if curr_chapter == nil then + curr_chapter = { + title = booknote[1].chapter, + entries = {} + } + elseif curr_chapter.title ~= booknote[1].chapter then + table.insert(chapters, curr_chapter) + curr_chapter = { + title = booknote[1].chapter, + entries = {} + } + end + table.insert(curr_chapter.entries, booknote[1]) + end + if curr_chapter ~= nil then + table.insert(chapters, curr_chapter) + end + booknotes.chapters = chapters + booknotes.entries = nil + return booknotes +end + +function HtmlExporter:export(t) + local title + local path = self:getFilePath(t) + if #t == 1 then + title = t[1].title + else + title = "All Books" + end + local file = io.open(path, "w") + local template = slt2.loadfile(self.path .. "/template/note.tpl") + local clipplings = {} + for _, booknotes in ipairs(t) do + table.insert(clipplings, format(booknotes)) + end + if not file then return false end + local content = slt2.render(template, { + clippings=clipplings, + document_title = title, + version = self:getVersion(), + timestamp = self:getTimeStamp(), + logger = logger + }) + file:write(content) + file:close() + return true +end + +return HtmlExporter diff --git a/plugins/exporter.koplugin/target/joplin.lua b/plugins/exporter.koplugin/target/joplin.lua new file mode 100644 index 000000000..5e086e747 --- /dev/null +++ b/plugins/exporter.koplugin/target/joplin.lua @@ -0,0 +1,364 @@ +local BD = require("ui/bidi") +local InfoMessage = require("ui/widget/infomessage") +local InputDialog = require("ui/widget/inputdialog") +local UIManager = require("ui/uimanager") +local http = require("socket.http") +local json = require("json") +local logger = require("logger") +local ltn12 = require("ltn12") +local socketutil = require("socketutil") +local T = require("ffi/util").template +local _ = require("gettext") + +-- joplin exporter +local JoplinExporter = require("base"):new { + name = "joplin", + is_remote = true, + notebook_name = _("KOReader Notes"), +} + +local function makeRequest(url, method, request_body) + local sink = {} + local request_body_json = json.encode(request_body) + local source = ltn12.source.string(request_body_json) + socketutil:set_timeout(socketutil.LARGE_BLOCK_TIMEOUT, socketutil.LARGE_TOTAL_TIMEOUT) + http.request{ + url = url, + method = method, + sink = ltn12.sink.table(sink), + source = source, + headers = { + ["Content-Length"] = #request_body_json, + ["Content-Type"] = "application/json" + }, + } + socketutil:reset_timeout() + + if not sink[1] then + return nil, "No response from Joplin Server" + end + + local response = json.decode(sink[1]) + + if not response then + return nil, "Unknown response from Joplin Server" + elseif response.error then + return nil, response.error + end + + return response +end + +local function ping(ip, port) + local sink = {} + http.request{ + url = "http://"..ip..":"..port.."/ping", + method = "GET", + sink = ltn12.sink.table(sink) + } + + if sink[1] == "JoplinClipperServer" then + return true + else + return false + end +end + +local function prepareNote(booknotes) + local note = "" + for _, clipping in ipairs(booknotes) do + local entry = clipping[1] + if entry.chapter then + note = note .. "\n\t*" .. entry.chapter .. "*\n\n * * *" + end + + note = note .. os.date("%Y-%m-%d %H:%M:%S \n", entry.time) + note = note .. entry.text + if entry.note then + note = note .. "\n---\n" .. entry.note + end + note = note .. "\n * * *\n" + end + return note +end + +-- If successful returns id of found note. +function JoplinExporter:findNoteByTitle(title, notebook_id) + local url_base = string.format("http://%s:%s/notes?token=%s&fields=id,title,parent_id&page=", + self.settings.ip, self.settings.port, self.settings.token) + + local page = 1 + local url, has_more + + repeat + url = url_base..page + local notes, err = makeRequest(url, "GET") + if not notes then + logger.warn("Joplin findNoteByTitle error", err) + return + end + has_more = notes.has_more + for _, note in ipairs(notes.items) do + if note.title == title and note.parent_id == notebook_id then + return note.id + end + end + page = page + 1 + until not has_more + return +end + +-- If successful returns id of found notebook (folder). +function JoplinExporter:findNotebookByTitle(title) + local url_base = string.format("http://%s:%s/folders?token=%s&query=title&page=", + self.settings.ip, self.settings.port, self.settings.token, title) + + local page = 1 + local url, has_more + + repeat + url = url_base .. page + local folders, err = makeRequest(url, "GET") + if not folders then + logger.warn("Joplin findNotebookByTitle error", err) + return + end + has_more = folders.has_more + for _, folder in ipairs(folders.items) do + if folder.title == title then + return folder.id + end + end + page = page + 1 + until not has_more + return +end + +-- returns true if the notebook exists +function JoplinExporter:notebookExist(title) + local url = string.format("http://%s:%s/folders?token=%s", + self.settings.ip, self.settings.port, self.settings.token) + local response, err = makeRequest(url, "GET") + if not response then + logger.warn("Joplin notebookExist error", err) + end + + if not response.items or type(response.items) ~= "table" then + return false + end + + for i, notebook in ipairs(response.items) do + if notebook.title == title then return true end + end + return false +end + +-- If successful returns id of created notebook (folder). +function JoplinExporter:createNotebook(title, created_time) + local request_body = { + title = title, + created_time = created_time + } + local url = string.format("http://%s:%s/folders?token=%s", + self.settings.ip, self.settings.port, self.settings.token) + + local response, err = makeRequest(url, "POST", request_body) + if not response then + logger.warn("Joplin createNotebook error", err) + return + end + return response.id +end + +-- If successful returns id of created note. +function JoplinExporter:createNote(title, note, parent_id, created_time) + local request_body = { + title = title, + body = note, + parent_id = parent_id, + created_time = created_time + } + local url = string.format("http://%s:%s/notes?token=%s", + self.settings.ip, self.settings.port, self.settings.token) + + local response, err = makeRequest(url, "POST", request_body) + if not response then + logger.warn("Joplin createNote error", err) + return + end + return response.id +end + +-- If successful returns id of updated note. +function JoplinExporter:updateNote(note, note_id) + local request_body = { + body = note + } + + local url = string.format("http://%s:%s/notes/%s?token=%s", + self.settings.ip, self.settings.port, note_id, self.settings.token) + + local response, err = makeRequest(url, "PUT", request_body) + if not response then + logger.warn("Joplin updateNote error", err) + return + end + return response.id +end + +function JoplinExporter:isReadyToExport() + return self.settings.ip and self.settings.port and self.settings.token +end + +function JoplinExporter:getMenuTable() + return { + text = _("Joplin"), + checked_func = function() return self:isEnabled() end, + sub_item_table = { + { + text = _("Set Joplin IP and Port"), + keep_menu_open = true, + callback = function() + local MultiInputDialog = require("ui/widget/multiinputdialog") + local url_dialog + url_dialog = MultiInputDialog:new { + title = _("Set Joplin IP and port number"), + fields = { + { + text = self.settings.ip, + input_type = "string" + }, + { + text = self.settings.port, + input_type = "number" + } + }, + buttons = { + { + { + text = _("Cancel"), + callback = function() + UIManager:close(url_dialog) + end + }, + { + text = _("OK"), + callback = function() + local fields = url_dialog:getFields() + local ip = fields[1] + local port = tonumber(fields[2]) + if ip ~= "" then + if port and port < 65355 then + self.settings.ip = ip + self.settings.port = port + self:saveSettings() + end + end + UIManager:close(url_dialog) + end + } + } + } + } + UIManager:show(url_dialog) + url_dialog:onShowKeyboard() + end + }, + { + text = _("Set authorization token"), + keep_menu_open = true, + callback = function() + local auth_dialog + auth_dialog = InputDialog:new { + title = _("Set authorization token for Joplin"), + input = self.settings.token, + buttons = { + { + { + text = _("Cancel"), + callback = function() + UIManager:close(auth_dialog) + end + }, + { + text = _("Set token"), + callback = function() + self.settings.token = auth_dialog:getInputText() + self:saveSettings() + UIManager:close(auth_dialog) + end + } + } + } + } + UIManager:show(auth_dialog) + auth_dialog:onShowKeyboard() + end + }, + { + text = _("Export to Joplin"), + checked_func = function() return self:isEnabled() end, + callback = function() self:toggleEnabled() end, + }, + { + text = _("Help"), + keep_menu_open = true, + callback = function() + UIManager:show(InfoMessage:new { + text = T(_([[You can enter your auth token on your computer by saving an empty token. Then quit KOReader, edit the exporter.joplin_token field in %1/settings.reader.lua after creating a backup, and restart KOReader once you're done. + +To export to Joplin, you must forward the IP and port used by this plugin to the localhost:port on which Joplin is listening. This can be done with socat or a similar program. For example: + +For Windows: netsh interface portproxy add v4tov4 listenaddress=0.0.0.0 listenport=41185 connectaddress=localhost connectport=41184 + +For Linux: $socat tcp-listen:41185,reuseaddr,fork tcp:localhost:41184 + +For more information, please visit https://github.com/koreader/koreader/wiki/Highlight-export.]]) + , BD.dirpath("example")) + }) + end + } + } + } +end + +function JoplinExporter:export(t) + if not self:isReadyToExport() then return false end + + if not ping(self.settings.ip, self.settings.port) then + logger.warn("Cannot reach Joplin server") + return false + end + + if not self:notebookExist(self.notebook_name) then + local notebook = self:createNotebook(self.notebook_name) + if notebook then + logger.info("Joplin: created new notebook", + "name", self.notebook_name, "id", notebook) + self.settings.notebook_guid = notebook + self:saveSettings() + else + logger.warn("Joplin: unable to create new notebook") + return false + end + end + + local notebook_id = self.settings.notebook_guid + for _, booknotes in pairs(t) do + local note = prepareNote(booknotes) + local note_id = self:findNoteByTitle(booknotes.title, notebook_id) + local response + if note_id then + response = self:updateNote(note, note_id) + else + response = self:createNote(booknotes.title, note, notebook_id) + end + if not response then + logger.warn("Cannot export to Joplin") + return false + end + end + return true +end + +return JoplinExporter diff --git a/plugins/exporter.koplugin/target/json.lua b/plugins/exporter.koplugin/target/json.lua new file mode 100644 index 000000000..07a8e8093 --- /dev/null +++ b/plugins/exporter.koplugin/target/json.lua @@ -0,0 +1,49 @@ +local json = require("json") + +-- json exporter +local JsonExporter = require("base"):new { + name = "json", +} + +local function format(booknotes) + local t = { + title = booknotes.title, + author = booknotes.author, + entries = {}, + exported = booknotes.exported, + file = booknotes.file + } + for _, entry in ipairs(booknotes) do + table.insert(t.entries, entry[1]) + end + return t +end + +function JsonExporter:export(t) + local exportable + local timestamp = self.timestamp or os.time() + local path = self:getFilePath(t) + if #t == 1 then + exportable = format(t[1]) + exportable.created_on = timestamp + exportable.version = self:getVersion() + else + local documents = {} + for _, booknotes in ipairs(t) do + table.insert(documents, format(booknotes)) + end + exportable = { + created_on = timestamp, + version = self:getVersion(), + documents = documents + } + end + local file = io.open(path, "w") + if not file then return false end + file:write(json.encode(exportable)) + file:write("\n") + file:close() + return true +end + +return JsonExporter diff --git a/plugins/exporter.koplugin/target/readwise.lua b/plugins/exporter.koplugin/target/readwise.lua new file mode 100644 index 000000000..6b3f16144 --- /dev/null +++ b/plugins/exporter.koplugin/target/readwise.lua @@ -0,0 +1,135 @@ +local InputDialog = require("ui/widget/inputdialog") +local UIManager = require("ui/uimanager") +local http = require("socket.http") +local json = require("json") +local logger = require("logger") +local ltn12 = require("ltn12") +local socket = require("socket") +local socketutil = require("socketutil") +local _ = require("gettext") + +-- readwise exporter +local ReadwiseExporter = require("base"):new { + name = "readwise", + is_remote = true, +} + +local function makeRequest(endpoint, method, request_body, token) + local sink = {} + local request_body_json = json.encode(request_body) + local source = ltn12.source.string(request_body_json) + socketutil:set_timeout(socketutil.LARGE_BLOCK_TIMEOUT, socketutil.LARGE_TOTAL_TIMEOUT) + local request = { + url = "https://readwise.io/api/v2/" .. endpoint, + method = method, + sink = ltn12.sink.table(sink), + source = source, + headers = { + ["Content-Length"] = #request_body_json, + ["Content-Type"] = "application/json", + ["Authorization"] = "Token " .. token + }, + } + local code, _, status = socket.skip(1, http.request(request)) + socketutil:reset_timeout() + + if code ~= 200 then + logger.warn("Readwise: HTTP response code <> 200. Response status: ", status) + return nil, status + end + + local response = json.decode(sink[1]) + return response +end + +function ReadwiseExporter:isReadyToExport() + if self.settings.token then return true end + return false +end + +function ReadwiseExporter:getMenuTable() + return { + text = _("Readwise"), + checked_func = function() return self:isEnabled() end, + sub_item_table = { + { + text = _("Set authorization token"), + keep_menu_open = true, + callback = function() + local auth_dialog + auth_dialog = InputDialog:new { + title = _("Set authorization token for Readwise"), + input = self.settings.token, + buttons = { + { + { + text = _("Cancel"), + callback = function() + UIManager:close(auth_dialog) + end + }, + { + text = _("Set token"), + callback = function() + self.settings.token = auth_dialog:getInputText() + self:saveSettings() + UIManager:close(auth_dialog) + end + } + } + } + } + UIManager:show(auth_dialog) + auth_dialog:onShowKeyboard() + end + }, + { + text = _("Export to Readwise"), + checked_func = function() return self:isEnabled() end, + callback = function() self:toggleEnabled() end, + }, + + } + } +end + +function ReadwiseExporter:createHighlights(booknotes) + local highlights = {} + for _, chapter in ipairs(booknotes) do + for _, clipping in ipairs(chapter) do + local highlight = { + text = clipping.text, + title = booknotes.title, + author = booknotes.author ~= "" and booknotes.author or nil, -- optional author + source_type = "koreader", + category = "books", + note = clipping.note, + location = clipping.page, + location_type = "page", + highlighted_at = os.date("!%Y-%m-%dT%TZ", clipping.time), + } + table.insert(highlights, highlight) + end + end + + local result, err = makeRequest("highlights", "POST", { highlights = highlights }, self.settings.token) + if not result then + logger.warn("error creating highlights", err) + return false + end + + logger.dbg("createHighlights result", result) + return true +end + +function ReadwiseExporter:export(t) + if not self:isReadyToExport() then return false end + + for _, booknotes in ipairs(t) do + local ok = self:createHighlights(booknotes) + if not ok then return false end + end + return true +end + +return ReadwiseExporter diff --git a/plugins/exporter.koplugin/target/text.lua b/plugins/exporter.koplugin/target/text.lua new file mode 100644 index 000000000..50d992a28 --- /dev/null +++ b/plugins/exporter.koplugin/target/text.lua @@ -0,0 +1,46 @@ +local util = require("ffi/util") +local T = util.template +local _ = require("gettext") + +-- text exporter +local TextExporter = require("base"):new { + name = "text", + extension = "txt", +} + +function TextExporter:export(t) + -- Use wide_space to avoid crengine to treat it specially. + local wide_space = "\227\128\128" + local path = self:getFilePath(t) + local file = io.open(path, "a") + if not file then return false end + for __, booknotes in ipairs(t) do + if booknotes.title then + file:write(wide_space .. booknotes.title .. "\n" .. wide_space .. "\n") + end + for ___, entry in ipairs(booknotes) do + for ____, clipping in ipairs(entry) do + if clipping.chapter then + file:write(wide_space .. clipping.chapter .. "\n" .. wide_space .. "\n") + end + local text = T(_("-- Page: %1, added on %2\n"), clipping.page, os.date("%c", clipping.time)) + file:write(wide_space .. wide_space .. text) + if clipping.text then + file:write(clipping.text) + end + if clipping.note then + file:write("\n---\n" .. clipping.note) + end + if clipping.image then + file:write(_("")) + end + file:write("\n-=-=-=-=-=-\n") + end + end + file:write("\n") + end + file:close() + return true +end + +return TextExporter diff --git a/plugins/exporter.koplugin/note.tpl b/plugins/exporter.koplugin/template/note.tpl similarity index 92% rename from plugins/exporter.koplugin/note.tpl rename to plugins/exporter.koplugin/template/note.tpl index 83ace096e..60b1e770e 100644 --- a/plugins/exporter.koplugin/note.tpl +++ b/plugins/exporter.koplugin/template/note.tpl @@ -52,17 +52,18 @@ - #{= htmlescape(booknotes.title) }# + #{= htmlescape(document_title) }#
+ #{ for _, booknotes in ipairs(clippings) do }#

#{= htmlescape(booknotes.title) }#

#{= htmlescape(booknotes.author) }#
- #{ for _, chapter in ipairs(booknotes) do }# + #{ for _, chapter in ipairs(booknotes.chapters) do }# #{ if chapter.title then }#
#{= htmlescape(chapter.title) }#
#{ end }# - #{ for index, clipping in ipairs(chapter) do }# + #{ for index, clipping in ipairs(chapter.entries) do }#
@@ -82,6 +83,7 @@
#{ end }# #{ end }# + #{ end }#
diff --git a/plugins/exporter.koplugin/slt2.lua b/plugins/exporter.koplugin/template/slt2.lua similarity index 100% rename from plugins/exporter.koplugin/slt2.lua rename to plugins/exporter.koplugin/template/slt2.lua diff --git a/spec/unit/exporter_plugin_main_spec.lua b/spec/unit/exporter_plugin_main_spec.lua index 3d36ccf15..7a93fa7b9 100644 --- a/spec/unit/exporter_plugin_main_spec.lua +++ b/spec/unit/exporter_plugin_main_spec.lua @@ -1,15 +1,14 @@ describe("Exporter plugin module", function() - local readerui, match + local readerui local sample_clippings, sample_epub local DocumentRegistry, Screen setup(function() require("commonrequire") - match = require("luassert.match") local ReaderUI = require("apps/reader/readerui") DocumentRegistry = require("document/documentregistry") Screen = require("device").screen sample_epub = "spec/front/unit/data/juliet.epub" - readerui = ReaderUI:new{ + readerui = ReaderUI:new { dimen = Screen:getSize(), document = DocumentRegistry:openDocument(sample_epub), } @@ -21,7 +20,8 @@ describe("Exporter plugin module", function() ["page"] = 6, ["time"] = 1578946897, ["sort"] = "highlight", - ["text"] = "Some important stuff 1" + ["text"] = "Some important stuff 1", + ["drawer"] = "lighten" } }, [2] = { @@ -29,7 +29,8 @@ describe("Exporter plugin module", function() ["page"] = 13, ["time"] = 1578946903, ["sort"] = "highlight", - ["text"] = "Some important stuff 2" + ["text"] = "Some important stuff 2", + ["drawer"] = "lighten" } }, ["file"] = "path/to/title1", @@ -45,7 +46,8 @@ describe("Exporter plugin module", function() ["page"] = 233, ["time"] = 1578946918, ["sort"] = "highlight", - ["text"] = "Some important stuff 3" + ["text"] = "Some important stuff 3", + ["drawer"] = "lighten" } }, [2] = { @@ -54,6 +56,7 @@ describe("Exporter plugin module", function() ["time"] = 1578947501, ["sort"] = "highlight", ["text"] = "", + ["drawer"] = "lighten", ["image"] = { ["hash"] = "cb7b40a63afc89f0aa452f2b655877e6", ["png"] = "Binary Encoding of image" @@ -62,46 +65,32 @@ describe("Exporter plugin module", function() }, ["file"] = "path/to/title2", ["exported"] = { - }, + }, ["title"] = "Title2" }, - } + } end) teardown(function() readerui:onClose() end) - it("should write clippings to txt file", function () - local file_mock = mock( { - write = function() return end, - close = function() return end - }) - local old_io = _G.io - _G.io = mock({ - open = function(file, mode) - if file == readerui.exporter.text_clipping_file then - return file_mock - else - return old_io.open(file, mode) - end - end - }) - - readerui.exporter:exportBooknotesToTXT("Title1", sample_clippings.Title1) - assert.spy(io.open).was.called() - assert.spy(file_mock.write).was.called_with(match.is_ref(file_mock), "Some important stuff 1") - _G.io = old_io - + it("should write clippings to a timestamped txt file", function() + local timestamp = os.time() + readerui.exporter.targets["text"].timestamp = timestamp + local exportable = { sample_clippings.Title1 } + local file_path = readerui.exporter.targets["text"]:getFilePath(exportable) + readerui.exporter.targets["text"]:export(exportable) + local f = io.open(file_path, "r") + assert.is.truthy(string.find(f:read("*all"), "Some important stuff 1")) + f:close() + os.remove(file_path) end) - it("should not export booknotes with exported_stamp", function() - readerui.exporter.html_export = true - stub(readerui.exporter, "exportBooknotesToHTML") - readerui.exporter:exportClippings(sample_clippings) - assert.stub(readerui.exporter.exportBooknotesToHTML).was_called_with(match.is_truthy(), "Title2", match.is_truthy()) - assert.stub(readerui.exporter.exportBooknotesToHTML).was_not_called_with(match.is_truthy(), "Title1", match.is_truthy()) + it("should fail to export to non configured targets", function() + local ok = readerui.exporter.targets["joplin"]:export(sample_clippings) + assert.not_truthy(ok) + ok = readerui.exporter.targets["readwise"]:export(sample_clippings) + assert.not_truthy(ok) end) - - end)