From b7aaa0f24dd5142b05fff19b58913930f9910b28 Mon Sep 17 00:00:00 2001 From: Ozzie Isaacs Date: Thu, 2 Nov 2023 17:05:02 +0100 Subject: [PATCH 01/18] Add metadata change code --- cps/admin.py | 10 ++++- cps/config_sql.py | 40 ++++++++++++----- cps/constants.py | 5 +++ cps/helper.py | 82 +++++++++++++++++++++++++++++++--- cps/tasks/convert.py | 34 +++++++++++--- cps/templates/config_edit.html | 10 ++--- 6 files changed, 152 insertions(+), 29 deletions(-) diff --git a/cps/admin.py b/cps/admin.py index 045a9523..51fddbee 100644 --- a/cps/admin.py +++ b/cps/admin.py @@ -47,7 +47,7 @@ from . import constants, logger, helper, services, cli_param from . import db, calibre_db, ub, web_server, config, updater_thread, gdriveutils, \ kobo_sync_status, schedule from .helper import check_valid_domain, send_test_mail, reset_password, generate_password_hash, check_email, \ - valid_email, check_username + valid_email, check_username, get_calibre_binarypath from .gdriveutils import is_gdrive_ready, gdrive_support from .render_template import render_title_template, get_sidebar_config from .services.worker import WorkerThread @@ -1761,8 +1761,14 @@ def _configuration_update_helper(): constants.EXTENSIONS_UPLOAD = config.config_upload_formats.split(',') _config_string(to_save, "config_calibre") - _config_string(to_save, "config_converterpath") + _config_string(to_save, "config_binariesdir") _config_string(to_save, "config_kepubifypath") + if "config_binariesdir" in to_save: + calibre_status = helper.check_calibre(config.config_binariesdir) + if calibre_status: + return _configuration_result(calibre_status) + to_save["config_converterpath"] = get_calibre_binarypath("ebook-convert") + _config_string(to_save, "config_converterpath") reboot_required |= _config_int(to_save, "config_login_type") diff --git a/cps/config_sql.py b/cps/config_sql.py index 21644ccd..485c3fc2 100644 --- a/cps/config_sql.py +++ b/cps/config_sql.py @@ -34,6 +34,7 @@ except ImportError: from sqlalchemy.ext.declarative import declarative_base from . import constants, logger +from .subproc_wrapper import process_wait log = logger.create() @@ -138,6 +139,7 @@ class _Settings(_Base): config_kepubifypath = Column(String, default=None) config_converterpath = Column(String, default=None) + config_binariesdir = Column(String, default=None) config_calibre = Column(String) config_rarfile_location = Column(String, default=None) config_upload_formats = Column(String, default=','.join(constants.EXTENSIONS_UPLOAD)) @@ -184,9 +186,11 @@ class ConfigSQL(object): self.load() change = False - if self.config_converterpath == None: # pylint: disable=access-member-before-definition + + if self.config_binariesdir == None: # pylint: disable=access-member-before-definition change = True - self.config_converterpath = autodetect_calibre_binary() + self.config_binariesdir = autodetect_calibre_binaries() + self.config_converterpath = autodetect_converter_binary(self.config_binariesdir) if self.config_kepubifypath == None: # pylint: disable=access-member-before-definition change = True @@ -469,17 +473,32 @@ def _migrate_table(session, orm_class, secret_key=None): session.rollback() -def autodetect_calibre_binary(): +def autodetect_calibre_binaries(): if sys.platform == "win32": - calibre_path = ["C:\\program files\\calibre\\ebook-convert.exe", - "C:\\program files(x86)\\calibre\\ebook-convert.exe", - "C:\\program files(x86)\\calibre2\\ebook-convert.exe", - "C:\\program files\\calibre2\\ebook-convert.exe"] + calibre_path = ["C:\\program files\\calibre\\", + "C:\\program files(x86)\\calibre\\", + "C:\\program files(x86)\\calibre2\\", + "C:\\program files\\calibre2\\"] else: - calibre_path = ["/opt/calibre/ebook-convert"] + calibre_path = ["/opt/calibre/"] for element in calibre_path: - if os.path.isfile(element) and os.access(element, os.X_OK): - return element + supported_binary_paths = [os.path.join(element, binary) for binary in constants.SUPPORTED_CALIBRE_BINARIES.values()] + if all(os.path.isfile(binary_path) and os.access(binary_path, os.X_OK) for binary_path in supported_binary_paths): + values = [process_wait([binary_path, "--version"], pattern='\(calibre (.*)\)') for binary_path in supported_binary_paths] + if all(values): + version = values[0].group(1) + log.debug("calibre version %s", version) + return element + return "" + + +def autodetect_converter_binary(calibre_path): + if sys.platform == "win32": + converter_path = os.path.join(calibre_path, "ebook-convert.exe") + else: + converter_path = os.path.join(calibre_path, "ebook-convert") + if calibre_path and os.path.isfile(converter_path) and os.access(converter_path, os.X_OK): + return converter_path return "" @@ -521,6 +540,7 @@ def load_configuration(session, secret_key): session.commit() + def get_flask_session_key(_session): flask_settings = _session.query(_Flask_Settings).one_or_none() if flask_settings == None: diff --git a/cps/constants.py b/cps/constants.py index 18c4f1b1..d8842e78 100644 --- a/cps/constants.py +++ b/cps/constants.py @@ -156,6 +156,11 @@ EXTENSIONS_UPLOAD = {'txt', 'pdf', 'epub', 'kepub', 'mobi', 'azw', 'azw3', 'cbr' 'prc', 'doc', 'docx', 'fb2', 'html', 'rtf', 'lit', 'odt', 'mp3', 'mp4', 'ogg', 'opus', 'wav', 'flac', 'm4a', 'm4b'} +_extension = "" +if sys.platform == "win32": + _extension = ".exe" +SUPPORTED_CALIBRE_BINARIES = {binary:binary + _extension for binary in ["ebook-convert", "calibredb"]} + def has_flag(value, bit_flag): return bit_flag == (bit_flag & (value or 0)) diff --git a/cps/helper.py b/cps/helper.py index 0c526d01..cabc0363 100644 --- a/cps/helper.py +++ b/cps/helper.py @@ -54,8 +54,8 @@ from . import calibre_db, cli_param from .tasks.convert import TaskConvert from . import logger, config, db, ub, fs from . import gdriveutils as gd -from .constants import STATIC_DIR as _STATIC_DIR, CACHE_TYPE_THUMBNAILS, THUMBNAIL_TYPE_COVER, THUMBNAIL_TYPE_SERIES -from .subproc_wrapper import process_wait +from .constants import STATIC_DIR as _STATIC_DIR, CACHE_TYPE_THUMBNAILS, THUMBNAIL_TYPE_COVER, THUMBNAIL_TYPE_SERIES, SUPPORTED_CALIBRE_BINARIES +from .subproc_wrapper import process_wait, process_open from .services.worker import WorkerThread from .tasks.mail import TaskEmail from .tasks.thumbnail import TaskClearCoverThumbnailCache, TaskGenerateCoverThumbnails @@ -938,9 +938,10 @@ def save_cover(img, book_path): def do_download_file(book, book_format, client, data, headers): + book_name = data.name if config.config_use_google_drive: # startTime = time.time() - df = gd.getFileFromEbooksFolder(book.path, data.name + "." + book_format) + df = gd.getFileFromEbooksFolder(book.path, book_name + "." + book_format) # log.debug('%s', time.time() - startTime) if df: return gd.do_gdrive_download(df, headers) @@ -948,20 +949,47 @@ def do_download_file(book, book_format, client, data, headers): abort(404) else: filename = os.path.join(config.config_calibre_dir, book.path) - if not os.path.isfile(os.path.join(filename, data.name + "." + book_format)): + if not os.path.isfile(os.path.join(filename, book_name + "." + book_format)): # ToDo: improve error handling - log.error('File not found: %s', os.path.join(filename, data.name + "." + book_format)) + log.error('File not found: %s', os.path.join(filename, book_name + "." + book_format)) if client == "kobo" and book_format == "kepub": headers["Content-Disposition"] = headers["Content-Disposition"].replace(".kepub", ".kepub.epub") - response = make_response(send_from_directory(filename, data.name + "." + book_format)) + if config.config_binariesdir: + filename, book_name = do_calibre_export(book, book_format) + + response = make_response(send_from_directory(filename, book_name + "." + book_format)) # ToDo Check headers parameter for element in headers: response.headers[element[0]] = element[1] - log.info('Downloading file: {}'.format(os.path.join(filename, data.name + "." + book_format))) + log.info('Downloading file: {}'.format(os.path.join(filename, book_name + "." + book_format))) return response + +def do_calibre_export(book, book_format): + try: + quotes = [3, 5, 7, 9] + tmp_dir = os.path.join(gettempdir(), 'calibre_web') + if not os.path.isdir(tmp_dir): + os.mkdir(tmp_dir) + calibredb_binarypath = get_calibre_binarypath("calibredb") + opf_command = [calibredb_binarypath, 'export', '--dont-write-opf', str(book.id), + '--with-library', config.config_calibre_dir, '--to-dir', tmp_dir, + '--formats', book_format, "--template", "{} - {{authors}}".format(book.title)] + file_name = book.title + if len(book.authors) > 0: + file_name = file_name + ' - ' + book.authors[0].name + p = process_open(opf_command, quotes) + _, err = p.communicate() + if err: + log.error('Metadata embedder encountered an error: %s', err) + return tmp_dir, file_name + except OSError as ex: + # ToDo real error handling + log.error_or_exception(ex) + + ################################## @@ -984,6 +1012,35 @@ def check_unrar(unrar_location): return _('Error executing UnRar') +def check_calibre(calibre_location): + if not calibre_location: + return + + if not os.path.exists(calibre_location): + return _('Could not find the specified directory') + + if not os.path.isdir(calibre_location): + return _('Please specify a directory, not a file') + + try: + supported_binary_paths = [os.path.join(calibre_location, binary) for binary in SUPPORTED_CALIBRE_BINARIES.values()] + binaries_available=[os.path.isfile(binary_path) and os.access(binary_path, os.X_OK) for binary_path in supported_binary_paths] + if all(binaries_available): + values = [process_wait([binary_path, "--version"], pattern='\(calibre (.*)\)') for binary_path in supported_binary_paths] + if all(values): + version = values[0].group(1) + log.debug("calibre version %s", version) + else: + return _('Calibre binaries not viable') + else: + missing_binaries=[path for path, available in zip(SUPPORTED_CALIBRE_BINARIES.values(), binaries_available) if not available] + return _('Missing calibre binaries: %(missing)s', missing=", ".join(missing_binaries)) + + except (OSError, UnicodeDecodeError) as err: + log.error_or_exception(err) + return _('Error excecuting Calibre') + + def json_serial(obj): """JSON serializer for objects not serializable by default json code""" @@ -1047,6 +1104,17 @@ def get_download_link(book_id, book_format, client): abort(404) +def get_calibre_binarypath(binary): + binariesdir = config.config_binariesdir + if binariesdir: + try: + return os.path.join(binariesdir, SUPPORTED_CALIBRE_BINARIES[binary]) + except KeyError as ex: + log.error("Binary not supported by Calibre-Web: %s", SUPPORTED_CALIBRE_BINARIES[binary]) + pass + return "" + + def clear_cover_thumbnail_cache(book_id): if config.schedule_generate_book_covers: WorkerThread.add(None, TaskClearCoverThumbnailCache(book_id), hidden=True) diff --git a/cps/tasks/convert.py b/cps/tasks/convert.py index df6ae104..2bef9a20 100755 --- a/cps/tasks/convert.py +++ b/cps/tasks/convert.py @@ -19,8 +19,10 @@ import os import re from glob import glob -from shutil import copyfile +from shutil import copyfile, copyfileobj from markupsafe import escape +from tempfile import gettempdir +from time import time from sqlalchemy.exc import SQLAlchemyError from flask_babel import lazy_gettext as N_ @@ -35,10 +37,11 @@ from cps.ub import init_db_thread from cps.tasks.mail import TaskEmail from cps import gdriveutils - +from cps.constants import SUPPORTED_CALIBRE_BINARIES log = logger.create() +current_milli_time = lambda: int(round(time() * 1000)) class TaskConvert(CalibreTask): def __init__(self, file_path, book_id, task_message, settings, ereader_mail, user=None): @@ -61,15 +64,20 @@ class TaskConvert(CalibreTask): data = worker_db.get_book_format(self.book_id, self.settings['old_book_format']) df = gdriveutils.getFileFromEbooksFolder(cur_book.path, data.name + "." + self.settings['old_book_format'].lower()) - if df: + df_cover = gdriveutils.getFileFromEbooksFolder(cur_book.path, "cover.jpg") + if df and df_cover: datafile = os.path.join(config.config_calibre_dir, cur_book.path, data.name + "." + self.settings['old_book_format'].lower()) + datafile_cover = os.path.join(config.config_calibre_dir, + cur_book.path, "cover.jpg") if not os.path.exists(os.path.join(config.config_calibre_dir, cur_book.path)): os.makedirs(os.path.join(config.config_calibre_dir, cur_book.path)) df.GetContentFile(datafile) + df_cover.GetContentFile(datafile_cover) worker_db.session.close() else: + # ToDo Include cover in error handling error_message = _("%(format)s not found on Google Drive: %(fn)s", format=self.settings['old_book_format'], fn=data.name + "." + self.settings['old_book_format'].lower()) @@ -79,6 +87,7 @@ class TaskConvert(CalibreTask): filename = self._convert_ebook_format() if config.config_use_google_drive: os.remove(self.file_path + '.' + self.settings['old_book_format'].lower()) + os.remove(os.path.join(config.config_calibre_dir, cur_book.path, "cover.jpg")) if filename: if config.config_use_google_drive: @@ -225,15 +234,30 @@ class TaskConvert(CalibreTask): return check, None def _convert_calibre(self, file_path, format_old_ext, format_new_ext): + book_id = self.book_id try: # Linux py2.7 encode as list without quotes no empty element for parameters # linux py3.x no encode and as list without quotes no empty element for parameters # windows py2.7 encode as string with quotes empty element for parameters is okay # windows py 3.x no encode and as string with quotes empty element for parameters is okay # separate handling for windows and linux - quotes = [1, 2] + + quotes = [3, 5] + tmp_dir = os.path.join(gettempdir(), 'calibre_web') + if not os.path.isdir(tmp_dir): + os.mkdir(tmp_dir) + calibredb_binarypath = os.path.join(config.config_binariesdir, SUPPORTED_CALIBRE_BINARIES["calibredb"]) + opf_command = [calibredb_binarypath, 'show_metadata', '--as-opf', str(book_id), '--with-library', config.config_calibre_dir] + p = process_open(opf_command, quotes) + p.wait() + path_tmp_opf = os.path.join(tmp_dir, "metadata_" + str(current_milli_time()) + ".opf") + with open(path_tmp_opf, 'w') as fd: + copyfileobj(p.stdout, fd) + + quotes = [1, 2, 4, 6] command = [config.config_converterpath, (file_path + format_old_ext), - (file_path + format_new_ext)] + (file_path + format_new_ext), '--from-opf', path_tmp_opf, + '--cover', os.path.join(os.path.dirname(file_path), 'cover.jpg')] quotes_index = 3 if config.config_calibre: parameters = config.config_calibre.split(" ") diff --git a/cps/templates/config_edit.html b/cps/templates/config_edit.html index d101f960..2ec0575c 100644 --- a/cps/templates/config_edit.html +++ b/cps/templates/config_edit.html @@ -323,12 +323,12 @@
- +
- - - - + + + +
From cf6810db87ffd7d8f4ce48c707561e72789e3dd6 Mon Sep 17 00:00:00 2001 From: Ozzie Isaacs Date: Sat, 11 Nov 2023 14:48:59 +0100 Subject: [PATCH 02/18] Refactored get_temp_dir bugfixes export_metadata --- cps/file_helper.py | 26 +++++++++++++++++++++++ cps/gdrive.py | 6 ++---- cps/gdriveutils.py | 1 - cps/helper.py | 49 +++++++++++++++++++++++--------------------- cps/tasks/convert.py | 9 ++++---- cps/updater.py | 6 +++--- cps/uploader.py | 7 ++----- 7 files changed, 64 insertions(+), 40 deletions(-) create mode 100644 cps/file_helper.py diff --git a/cps/file_helper.py b/cps/file_helper.py new file mode 100644 index 00000000..717cbb7d --- /dev/null +++ b/cps/file_helper.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- + +# This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web) +# Copyright (C) 2023 OzzieIsaacs +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from tempfile import gettempdir +import os + +def get_temp_dir(): + tmp_dir = os.path.join(gettempdir(), 'calibre_web') + if not os.path.isdir(tmp_dir): + os.mkdir(tmp_dir) + return tmp_dir diff --git a/cps/gdrive.py b/cps/gdrive.py index 832350e1..4d110f83 100644 --- a/cps/gdrive.py +++ b/cps/gdrive.py @@ -23,7 +23,6 @@ import os import hashlib import json -import tempfile from uuid import uuid4 from time import time from shutil import move, copyfile @@ -34,6 +33,7 @@ from flask_login import login_required from . import logger, gdriveutils, config, ub, calibre_db, csrf from .admin import admin_required +from .file_helper import get_temp_dir gdrive = Blueprint('gdrive', __name__, url_prefix='/gdrive') log = logger.create() @@ -139,9 +139,7 @@ try: dbpath = os.path.join(config.config_calibre_dir, "metadata.db").encode() if not response['deleted'] and response['file']['title'] == 'metadata.db' \ and response['file']['md5Checksum'] != hashlib.md5(dbpath): # nosec - tmp_dir = os.path.join(tempfile.gettempdir(), 'calibre_web') - if not os.path.isdir(tmp_dir): - os.mkdir(tmp_dir) + tmp_dir = get_temp_dir() log.info('Database file updated') copyfile(dbpath, os.path.join(tmp_dir, "metadata.db_" + str(current_milli_time()))) diff --git a/cps/gdriveutils.py b/cps/gdriveutils.py index 08ead47d..b1d30596 100644 --- a/cps/gdriveutils.py +++ b/cps/gdriveutils.py @@ -34,7 +34,6 @@ except ImportError: from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.exc import OperationalError, InvalidRequestError, IntegrityError from sqlalchemy.orm.exc import StaleDataError -from sqlalchemy.sql.expression import text try: from httplib2 import __version__ as httplib2_version diff --git a/cps/helper.py b/cps/helper.py index cabc0363..013a6fc2 100644 --- a/cps/helper.py +++ b/cps/helper.py @@ -25,9 +25,9 @@ import re import shutil import socket from datetime import datetime, timedelta -from tempfile import gettempdir import requests import unidecode +from uuid import uuid4 from flask import send_from_directory, make_response, redirect, abort, url_for from flask_babel import gettext as _ @@ -60,6 +60,7 @@ from .services.worker import WorkerThread from .tasks.mail import TaskEmail from .tasks.thumbnail import TaskClearCoverThumbnailCache, TaskGenerateCoverThumbnails from .tasks.metadata_backup import TaskBackupMetadata +from .file_helper import get_temp_dir log = logger.create() @@ -921,10 +922,7 @@ def save_cover(img, book_path): return False, _("Only jpg/jpeg files are supported as coverfile") if config.config_use_google_drive: - tmp_dir = os.path.join(gettempdir(), 'calibre_web') - - if not os.path.isdir(tmp_dir): - os.mkdir(tmp_dir) + tmp_dir = get_temp_dir() ret, message = save_cover_from_filestorage(tmp_dir, "uploaded_cover.jpg", img) if ret is True: gd.uploadFileToEbooksFolder(os.path.join(book_path, 'cover.jpg').replace("\\", "/"), @@ -944,7 +942,13 @@ def do_download_file(book, book_format, client, data, headers): df = gd.getFileFromEbooksFolder(book.path, book_name + "." + book_format) # log.debug('%s', time.time() - startTime) if df: - return gd.do_gdrive_download(df, headers) + # ToDo check:!!!!!!!! + if config.config_binariesdir: + output = os.path.join(config.config_calibre_dir, book.path, data.name) + gd.ownloadFile(book.path, book_name + "." + book_format, output) + filename, download_name = do_calibre_export(book, book_format) + else: + return gd.do_gdrive_download(df, headers) else: abort(404) else: @@ -957,34 +961,33 @@ def do_download_file(book, book_format, client, data, headers): headers["Content-Disposition"] = headers["Content-Disposition"].replace(".kepub", ".kepub.epub") if config.config_binariesdir: - filename, book_name = do_calibre_export(book, book_format) + filename, download_name = do_calibre_export(book, book_format) + else: + download_name = book_name + + response = make_response(send_from_directory(filename, download_name + "." + book_format)) + # ToDo Check headers parameter + for element in headers: + response.headers[element[0]] = element[1] + log.info('Downloading file: {}'.format(os.path.join(filename, book_name + "." + book_format))) + return response - response = make_response(send_from_directory(filename, book_name + "." + book_format)) - # ToDo Check headers parameter - for element in headers: - response.headers[element[0]] = element[1] - log.info('Downloading file: {}'.format(os.path.join(filename, book_name + "." + book_format))) - return response def do_calibre_export(book, book_format): try: quotes = [3, 5, 7, 9] - tmp_dir = os.path.join(gettempdir(), 'calibre_web') - if not os.path.isdir(tmp_dir): - os.mkdir(tmp_dir) + tmp_dir = get_temp_dir() calibredb_binarypath = get_calibre_binarypath("calibredb") - opf_command = [calibredb_binarypath, 'export', '--dont-write-opf', str(book.id), - '--with-library', config.config_calibre_dir, '--to-dir', tmp_dir, - '--formats', book_format, "--template", "{} - {{authors}}".format(book.title)] - file_name = book.title - if len(book.authors) > 0: - file_name = file_name + ' - ' + book.authors[0].name + temp_file_name = str(uuid4()) + opf_command = [calibredb_binarypath, 'export', '--dont-write-opf', '--with-library', config.config_calibre_dir, + '--to-dir', tmp_dir, '--formats', book_format, "--template", "{}".format(temp_file_name), + str(book.id)] p = process_open(opf_command, quotes) _, err = p.communicate() if err: log.error('Metadata embedder encountered an error: %s', err) - return tmp_dir, file_name + return tmp_dir, temp_file_name except OSError as ex: # ToDo real error handling log.error_or_exception(ex) diff --git a/cps/tasks/convert.py b/cps/tasks/convert.py index 2bef9a20..61a2bda0 100755 --- a/cps/tasks/convert.py +++ b/cps/tasks/convert.py @@ -21,7 +21,6 @@ import re from glob import glob from shutil import copyfile, copyfileobj from markupsafe import escape -from tempfile import gettempdir from time import time from sqlalchemy.exc import SQLAlchemyError @@ -34,6 +33,7 @@ from cps.subproc_wrapper import process_open from flask_babel import gettext as _ from cps.kobo_sync_status import remove_synced_book from cps.ub import init_db_thread +from cps.file_helper import get_temp_dir from cps.tasks.mail import TaskEmail from cps import gdriveutils @@ -243,9 +243,10 @@ class TaskConvert(CalibreTask): # separate handling for windows and linux quotes = [3, 5] - tmp_dir = os.path.join(gettempdir(), 'calibre_web') - if not os.path.isdir(tmp_dir): - os.mkdir(tmp_dir) + tmp_dir = get_temp_dir() + #tmp_dir = os.path.join(gettempdir(), 'calibre_web') + #if not os.path.isdir(tmp_dir): + # os.mkdir(tmp_dir) calibredb_binarypath = os.path.join(config.config_binariesdir, SUPPORTED_CALIBRE_BINARIES["calibredb"]) opf_command = [calibredb_binarypath, 'show_metadata', '--as-opf', str(book_id), '--with-library', config.config_calibre_dir] p = process_open(opf_command, quotes) diff --git a/cps/updater.py b/cps/updater.py index 6d6e408f..4369e18f 100644 --- a/cps/updater.py +++ b/cps/updater.py @@ -25,13 +25,13 @@ import threading import time import zipfile from io import BytesIO -from tempfile import gettempdir - import requests + from flask_babel import format_datetime from flask_babel import gettext as _ from . import constants, logger # config, web_server +from .file_helper import gettempdir log = logger.create() @@ -85,7 +85,7 @@ class Updater(threading.Thread): z = zipfile.ZipFile(BytesIO(r.content)) self.status = 3 log.debug('Extracting zipfile') - tmp_dir = gettempdir() + tmp_dir = get_temp_dir() z.extractall(tmp_dir) folder_name = os.path.join(tmp_dir, z.namelist()[0])[:-1] if not os.path.isdir(folder_name): diff --git a/cps/uploader.py b/cps/uploader.py index 23dfc4a6..8f20762f 100644 --- a/cps/uploader.py +++ b/cps/uploader.py @@ -18,12 +18,12 @@ import os import hashlib -from tempfile import gettempdir from flask_babel import gettext as _ from . import logger, comic, isoLanguages from .constants import BookMeta from .helper import split_authors +from .file_helper import get_temp_dir log = logger.create() @@ -249,10 +249,7 @@ def get_magick_version(): def upload(uploadfile, rar_excecutable): - tmp_dir = os.path.join(gettempdir(), 'calibre_web') - - if not os.path.isdir(tmp_dir): - os.mkdir(tmp_dir) + tmp_dir = get_temp_dir() filename = uploadfile.filename filename_root, file_extension = os.path.splitext(filename) From 7fd1d10fca3220329d6d762d3851486150e93405 Mon Sep 17 00:00:00 2001 From: Ozzie Isaacs Date: Sat, 11 Nov 2023 15:26:05 +0100 Subject: [PATCH 03/18] Implement gdrive metadata on download --- cps/helper.py | 9 +++++---- cps/updater.py | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/cps/helper.py b/cps/helper.py index 013a6fc2..ab3123ec 100644 --- a/cps/helper.py +++ b/cps/helper.py @@ -942,10 +942,12 @@ def do_download_file(book, book_format, client, data, headers): df = gd.getFileFromEbooksFolder(book.path, book_name + "." + book_format) # log.debug('%s', time.time() - startTime) if df: - # ToDo check:!!!!!!!! if config.config_binariesdir: - output = os.path.join(config.config_calibre_dir, book.path, data.name) - gd.ownloadFile(book.path, book_name + "." + book_format, output) + output_path = os.path.join(config.config_calibre_dir, book.path) + if not os.path.exists(output_path): + os.makedirs(output_path) + output = os.path.join(config.config_calibre_dir, book.path, book_name + "." + book_format) + gd.downloadFile(book.path, book_name + "." + book_format, output) filename, download_name = do_calibre_export(book, book_format) else: return gd.do_gdrive_download(df, headers) @@ -973,7 +975,6 @@ def do_download_file(book, book_format, client, data, headers): return response - def do_calibre_export(book, book_format): try: quotes = [3, 5, 7, 9] diff --git a/cps/updater.py b/cps/updater.py index 4369e18f..8150aa0d 100644 --- a/cps/updater.py +++ b/cps/updater.py @@ -31,7 +31,7 @@ from flask_babel import format_datetime from flask_babel import gettext as _ from . import constants, logger # config, web_server -from .file_helper import gettempdir +from .file_helper import get_temp_dir log = logger.create() From 794cd354ca8be3eb307c7c1dd7e913b81f0e8c81 Mon Sep 17 00:00:00 2001 From: Ozzie Isaacs Date: Sun, 12 Nov 2023 11:02:30 +0100 Subject: [PATCH 04/18] Added ToDo's --- cps/helper.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/cps/helper.py b/cps/helper.py index ab3123ec..c06ab402 100644 --- a/cps/helper.py +++ b/cps/helper.py @@ -949,6 +949,7 @@ def do_download_file(book, book_format, client, data, headers): output = os.path.join(config.config_calibre_dir, book.path, book_name + "." + book_format) gd.downloadFile(book.path, book_name + "." + book_format, output) filename, download_name = do_calibre_export(book, book_format) + # ToDo: delete path in calibre-folder structure else: return gd.do_gdrive_download(df, headers) else: @@ -1027,17 +1028,21 @@ def check_calibre(calibre_location): return _('Please specify a directory, not a file') try: - supported_binary_paths = [os.path.join(calibre_location, binary) for binary in SUPPORTED_CALIBRE_BINARIES.values()] - binaries_available=[os.path.isfile(binary_path) and os.access(binary_path, os.X_OK) for binary_path in supported_binary_paths] + supported_binary_paths = [os.path.join(calibre_location, binary) + for binary in SUPPORTED_CALIBRE_BINARIES.values()] + binaries_available=[os.path.isfile(binary_path) and os.access(binary_path, os.X_OK) + for binary_path in supported_binary_paths] if all(binaries_available): - values = [process_wait([binary_path, "--version"], pattern='\(calibre (.*)\)') for binary_path in supported_binary_paths] + values = [process_wait([binary_path, "--version"], pattern='\(calibre (.*)\)') + for binary_path in supported_binary_paths] if all(values): version = values[0].group(1) log.debug("calibre version %s", version) else: return _('Calibre binaries not viable') else: - missing_binaries=[path for path, available in zip(SUPPORTED_CALIBRE_BINARIES.values(), binaries_available) if not available] + missing_binaries=[path for path, available in + zip(SUPPORTED_CALIBRE_BINARIES.values(), binaries_available) if not available] return _('Missing calibre binaries: %(missing)s', missing=", ".join(missing_binaries)) except (OSError, UnicodeDecodeError) as err: From 7fbbb85f475e00af351bff8f27172747c321f28e Mon Sep 17 00:00:00 2001 From: Ozzie Isaacs Date: Mon, 4 Dec 2023 19:26:43 +0100 Subject: [PATCH 05/18] Temp folder is deleted on regular base --- cps/file_helper.py | 6 ++++++ cps/schedule.py | 6 ++++++ cps/tasks/tempFolder.py | 47 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 59 insertions(+) create mode 100644 cps/tasks/tempFolder.py diff --git a/cps/file_helper.py b/cps/file_helper.py index 717cbb7d..7c3e5291 100644 --- a/cps/file_helper.py +++ b/cps/file_helper.py @@ -18,9 +18,15 @@ from tempfile import gettempdir import os +import shutil def get_temp_dir(): tmp_dir = os.path.join(gettempdir(), 'calibre_web') if not os.path.isdir(tmp_dir): os.mkdir(tmp_dir) return tmp_dir + + +def del_temp_dir(): + tmp_dir = os.path.join(gettempdir(), 'calibre_web') + shutil.rmtree(tmp_dir) diff --git a/cps/schedule.py b/cps/schedule.py index 05367e99..c1c4aafc 100644 --- a/cps/schedule.py +++ b/cps/schedule.py @@ -21,6 +21,7 @@ import datetime from . import config, constants from .services.background_scheduler import BackgroundScheduler, CronTrigger, use_APScheduler from .tasks.database import TaskReconnectDatabase +from .tasks.tempFolder import TaskDeleteTempFolder from .tasks.thumbnail import TaskGenerateCoverThumbnails, TaskGenerateSeriesThumbnails, TaskClearCoverThumbnailCache from .services.worker import WorkerThread from .tasks.metadata_backup import TaskBackupMetadata @@ -31,6 +32,9 @@ def get_scheduled_tasks(reconnect=True): if reconnect: tasks.append([lambda: TaskReconnectDatabase(), 'reconnect', False]) + # Delete temp folder + tasks.append([lambda: TaskDeleteTempFolder(), 'delete temp', False]) + # Generate metadata.opf file for each changed book if config.schedule_metadata_backup: tasks.append([lambda: TaskBackupMetadata("en"), 'backup metadata', False]) @@ -86,6 +90,8 @@ def register_startup_tasks(): # Ignore tasks that should currently be running, as these will be added when registering scheduled tasks if constants.APP_MODE in ['development', 'test'] and not should_task_be_running(start, duration): scheduler.schedule_tasks_immediately(tasks=get_scheduled_tasks(False)) + else: + scheduler.schedule_tasks_immediately(tasks=[[lambda: TaskDeleteTempFolder(), 'delete temp', False]]) def should_task_be_running(start, duration): diff --git a/cps/tasks/tempFolder.py b/cps/tasks/tempFolder.py new file mode 100644 index 00000000..e740cd1e --- /dev/null +++ b/cps/tasks/tempFolder.py @@ -0,0 +1,47 @@ +# -*- coding: utf-8 -*- + +# This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web) +# Copyright (C) 2023 OzzieIsaacs +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from urllib.request import urlopen + +from flask_babel import lazy_gettext as N_ + +from cps import logger, file_helper +from cps.services.worker import CalibreTask + + +class TaskDeleteTempFolder(CalibreTask): + def __init__(self, task_message=N_('Delete temp folder contents')): + super(TaskDeleteTempFolder, self).__init__(task_message) + self.log = logger.create() + + def run(self, worker_thread): + try: + file_helper.del_temp_dir() + except FileNotFoundError: + pass + except (PermissionError, OSError) as e: + self.log.error("Error deleting temp folder: {}".format(e)) + self._handleSuccess() + + @property + def name(self): + return "Delete Temp Folder" + + @property + def is_cancellable(self): + return False From fd90d6e3754ff3004678e44a6c406f727eb36942 Mon Sep 17 00:00:00 2001 From: Ozzie Isaacs Date: Wed, 6 Dec 2023 19:20:42 +0100 Subject: [PATCH 06/18] Delete temp dir not shown in tasks overview Handle case no cover during convert ebook --- cps/schedule.py | 4 ++-- cps/tasks/convert.py | 11 +++++++---- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/cps/schedule.py b/cps/schedule.py index c1c4aafc..bf622b36 100644 --- a/cps/schedule.py +++ b/cps/schedule.py @@ -33,7 +33,7 @@ def get_scheduled_tasks(reconnect=True): tasks.append([lambda: TaskReconnectDatabase(), 'reconnect', False]) # Delete temp folder - tasks.append([lambda: TaskDeleteTempFolder(), 'delete temp', False]) + tasks.append([lambda: TaskDeleteTempFolder(), 'delete temp', True]) # Generate metadata.opf file for each changed book if config.schedule_metadata_backup: @@ -91,7 +91,7 @@ def register_startup_tasks(): if constants.APP_MODE in ['development', 'test'] and not should_task_be_running(start, duration): scheduler.schedule_tasks_immediately(tasks=get_scheduled_tasks(False)) else: - scheduler.schedule_tasks_immediately(tasks=[[lambda: TaskDeleteTempFolder(), 'delete temp', False]]) + scheduler.schedule_tasks_immediately(tasks=[[lambda: TaskDeleteTempFolder(), 'delete temp', True]]) def should_task_be_running(start, duration): diff --git a/cps/tasks/convert.py b/cps/tasks/convert.py index 61a2bda0..5bee0773 100755 --- a/cps/tasks/convert.py +++ b/cps/tasks/convert.py @@ -49,6 +49,7 @@ class TaskConvert(CalibreTask): self.file_path = file_path self.book_id = book_id self.title = "" + self.has_cover = None self.settings = settings self.ereader_mail = ereader_mail self.user = user @@ -161,7 +162,8 @@ class TaskConvert(CalibreTask): if not os.path.exists(config.config_converterpath): self._handleError(N_("Calibre ebook-convert %(tool)s not found", tool=config.config_converterpath)) return - check, error_message = self._convert_calibre(file_path, format_old_ext, format_new_ext) + has_cover = local_db.get_book(book_id).has_cover + check, error_message = self._convert_calibre(file_path, format_old_ext, format_new_ext, has_cover) if check == 0: cur_book = local_db.get_book(book_id) @@ -233,7 +235,7 @@ class TaskConvert(CalibreTask): folder=os.path.dirname(file_path)) return check, None - def _convert_calibre(self, file_path, format_old_ext, format_new_ext): + def _convert_calibre(self, file_path, format_old_ext, format_new_ext, has_cover): book_id = self.book_id try: # Linux py2.7 encode as list without quotes no empty element for parameters @@ -257,8 +259,9 @@ class TaskConvert(CalibreTask): quotes = [1, 2, 4, 6] command = [config.config_converterpath, (file_path + format_old_ext), - (file_path + format_new_ext), '--from-opf', path_tmp_opf, - '--cover', os.path.join(os.path.dirname(file_path), 'cover.jpg')] + (file_path + format_new_ext), '--from-opf', path_tmp_opf] + if has_cover: + command.extend(['--cover', os.path.join(os.path.dirname(file_path), 'cover.jpg')]) quotes_index = 3 if config.config_calibre: parameters = config.config_calibre.split(" ") From 90ad570578ba8d2aad45c61e79ff26519e8ceba0 Mon Sep 17 00:00:00 2001 From: Ozzie Isaacs Date: Thu, 7 Dec 2023 16:22:34 +0100 Subject: [PATCH 07/18] Show only folders for selecting converter binaries --- cps/templates/config_edit.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cps/templates/config_edit.html b/cps/templates/config_edit.html index 2ec0575c..736062f4 100644 --- a/cps/templates/config_edit.html +++ b/cps/templates/config_edit.html @@ -327,7 +327,7 @@
- +
From 2334e8f9c958f8f933f92dbbece197dda6dd3f27 Mon Sep 17 00:00:00 2001 From: Ozzie Isaacs Date: Thu, 7 Dec 2023 16:47:10 +0100 Subject: [PATCH 08/18] Refactored calibre executable detection for better error messages --- cps/helper.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/cps/helper.py b/cps/helper.py index c06ab402..a8acc750 100644 --- a/cps/helper.py +++ b/cps/helper.py @@ -1030,9 +1030,9 @@ def check_calibre(calibre_location): try: supported_binary_paths = [os.path.join(calibre_location, binary) for binary in SUPPORTED_CALIBRE_BINARIES.values()] - binaries_available=[os.path.isfile(binary_path) and os.access(binary_path, os.X_OK) - for binary_path in supported_binary_paths] - if all(binaries_available): + binaries_available = [os.path.isfile(binary_path) for binary_path in supported_binary_paths] + binaries_executable = [os.access(binary_path, os.X_OK) for binary_path in supported_binary_paths] + if all(binaries_available) and all(binaries_executable): values = [process_wait([binary_path, "--version"], pattern='\(calibre (.*)\)') for binary_path in supported_binary_paths] if all(values): @@ -1041,9 +1041,17 @@ def check_calibre(calibre_location): else: return _('Calibre binaries not viable') else: + ret_val = [] missing_binaries=[path for path, available in zip(SUPPORTED_CALIBRE_BINARIES.values(), binaries_available) if not available] - return _('Missing calibre binaries: %(missing)s', missing=", ".join(missing_binaries)) + + missing_perms=[path for path, available in + zip(SUPPORTED_CALIBRE_BINARIES.values(), binaries_executable) if not available] + if missing_binaries: + ret_val.append(_('Missing calibre binaries: %(missing)s', missing=", ".join(missing_binaries))) + if missing_perms: + ret_val.append(_('Missing executable permissions: %(missing)s', missing=", ".join(missing_perms))) + return ", ".join(ret_val) except (OSError, UnicodeDecodeError) as err: log.error_or_exception(err) From d341faf204a73ecfe17c988cc43cc50a9b4cbbb0 Mon Sep 17 00:00:00 2001 From: Ozzie Isaacs Date: Sat, 9 Dec 2023 09:36:28 +0100 Subject: [PATCH 09/18] Make Version setuptools compatible and still have the "Beta" in the User interface --- cps/about.py | 4 ++-- cps/admin.py | 2 +- cps/cli.py | 4 ++-- cps/constants.py | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cps/about.py b/cps/about.py index 7b6cc71a..1d081fe2 100644 --- a/cps/about.py +++ b/cps/about.py @@ -49,9 +49,9 @@ sorted_modules = OrderedDict((sorted(modules.items(), key=lambda x: x[0].casefol def collect_stats(): if constants.NIGHTLY_VERSION[0] == "$Format:%H$": - calibre_web_version = constants.STABLE_VERSION['version'] + calibre_web_version = constants.STABLE_VERSION['version'].replace("b", " Beta") else: - calibre_web_version = (constants.STABLE_VERSION['version'] + ' - ' + calibre_web_version = (constants.STABLE_VERSION['version'].replace("b", " Beta") + ' - ' + constants.NIGHTLY_VERSION[0].replace('%', '%%') + ' - ' + constants.NIGHTLY_VERSION[1].replace('%', '%%')) diff --git a/cps/admin.py b/cps/admin.py index 51fddbee..a933f8bc 100644 --- a/cps/admin.py +++ b/cps/admin.py @@ -214,7 +214,7 @@ def admin(): form_date += timedelta(hours=int(commit[20:22]), minutes=int(commit[23:])) commit = format_datetime(form_date - tz, format='short') else: - commit = version['version'] + commit = version['version'].replace("b", " Beta") all_user = ub.session.query(ub.User).all() # email_settings = mail_config.get_mail_settings() diff --git a/cps/cli.py b/cps/cli.py index e9b97b9d..855ad899 100644 --- a/cps/cli.py +++ b/cps/cli.py @@ -29,8 +29,8 @@ from .constants import DEFAULT_SETTINGS_FILE, DEFAULT_GDRIVE_FILE def version_info(): if _NIGHTLY_VERSION[1].startswith('$Format'): - return "Calibre-Web version: %s - unknown git-clone" % _STABLE_VERSION['version'] - return "Calibre-Web version: %s -%s" % (_STABLE_VERSION['version'], _NIGHTLY_VERSION[1]) + return "Calibre-Web version: %s - unknown git-clone" % _STABLE_VERSION['version'].replace("b", " Beta") + return "Calibre-Web version: %s -%s" % (_STABLE_VERSION['version'].replace("b", " Beta"), _NIGHTLY_VERSION[1]) class CliParameter(object): diff --git a/cps/constants.py b/cps/constants.py index d8842e78..09f5cf53 100644 --- a/cps/constants.py +++ b/cps/constants.py @@ -173,7 +173,7 @@ def selected_roles(dictionary): BookMeta = namedtuple('BookMeta', 'file_path, extension, title, author, cover, description, tags, series, ' 'series_id, languages, publisher, pubdate, identifiers') -STABLE_VERSION = {'version': '0.6.22 Beta'} +STABLE_VERSION = {'version': '0.6.22b'} NIGHTLY_VERSION = dict() NIGHTLY_VERSION[0] = '$Format:%H$' From 1086296d1dd44854493fd8c9d673ac7fd698f22b Mon Sep 17 00:00:00 2001 From: Ozzie Isaacs Date: Sat, 9 Dec 2023 11:23:26 +0100 Subject: [PATCH 10/18] Make embed metadata configurable --- cps/admin.py | 1 + cps/config_sql.py | 1 + cps/helper.py | 5 ++--- cps/tasks/convert.py | 33 ++++++++++++++------------------- cps/templates/config_edit.html | 4 ++++ 5 files changed, 22 insertions(+), 22 deletions(-) diff --git a/cps/admin.py b/cps/admin.py index a933f8bc..3af3e422 100644 --- a/cps/admin.py +++ b/cps/admin.py @@ -1745,6 +1745,7 @@ def _configuration_update_helper(): _config_checkbox_int(to_save, "config_uploading") _config_checkbox_int(to_save, "config_unicode_filename") + _config_checkbox_int(to_save, "config_embed_metadata") # Reboot on config_anonbrowse with enabled ldap, as decoraters are changed in this case reboot_required |= (_config_checkbox_int(to_save, "config_anonbrowse") and config.config_login_type == constants.LOGIN_LDAP) diff --git a/cps/config_sql.py b/cps/config_sql.py index 485c3fc2..ac8f7b5d 100644 --- a/cps/config_sql.py +++ b/cps/config_sql.py @@ -144,6 +144,7 @@ class _Settings(_Base): config_rarfile_location = Column(String, default=None) config_upload_formats = Column(String, default=','.join(constants.EXTENSIONS_UPLOAD)) config_unicode_filename = Column(Boolean, default=False) + config_embed_metadata = Column(Boolean, default=True) config_updatechannel = Column(Integer, default=constants.UPDATE_STABLE) diff --git a/cps/helper.py b/cps/helper.py index a8acc750..326e2355 100644 --- a/cps/helper.py +++ b/cps/helper.py @@ -942,14 +942,13 @@ def do_download_file(book, book_format, client, data, headers): df = gd.getFileFromEbooksFolder(book.path, book_name + "." + book_format) # log.debug('%s', time.time() - startTime) if df: - if config.config_binariesdir: + if config.config_binariesdir and config.config_embed_metadata: output_path = os.path.join(config.config_calibre_dir, book.path) if not os.path.exists(output_path): os.makedirs(output_path) output = os.path.join(config.config_calibre_dir, book.path, book_name + "." + book_format) gd.downloadFile(book.path, book_name + "." + book_format, output) filename, download_name = do_calibre_export(book, book_format) - # ToDo: delete path in calibre-folder structure else: return gd.do_gdrive_download(df, headers) else: @@ -963,7 +962,7 @@ def do_download_file(book, book_format, client, data, headers): if client == "kobo" and book_format == "kepub": headers["Content-Disposition"] = headers["Content-Disposition"].replace(".kepub", ".kepub.epub") - if config.config_binariesdir: + if config.config_binariesdir and config.config_embed_metadata: filename, download_name = do_calibre_export(book, book_format) else: download_name = book_name diff --git a/cps/tasks/convert.py b/cps/tasks/convert.py index 5bee0773..a39b3f61 100755 --- a/cps/tasks/convert.py +++ b/cps/tasks/convert.py @@ -238,28 +238,23 @@ class TaskConvert(CalibreTask): def _convert_calibre(self, file_path, format_old_ext, format_new_ext, has_cover): book_id = self.book_id try: - # Linux py2.7 encode as list without quotes no empty element for parameters - # linux py3.x no encode and as list without quotes no empty element for parameters - # windows py2.7 encode as string with quotes empty element for parameters is okay - # windows py 3.x no encode and as string with quotes empty element for parameters is okay - # separate handling for windows and linux - - quotes = [3, 5] - tmp_dir = get_temp_dir() - #tmp_dir = os.path.join(gettempdir(), 'calibre_web') - #if not os.path.isdir(tmp_dir): - # os.mkdir(tmp_dir) - calibredb_binarypath = os.path.join(config.config_binariesdir, SUPPORTED_CALIBRE_BINARIES["calibredb"]) - opf_command = [calibredb_binarypath, 'show_metadata', '--as-opf', str(book_id), '--with-library', config.config_calibre_dir] - p = process_open(opf_command, quotes) - p.wait() - path_tmp_opf = os.path.join(tmp_dir, "metadata_" + str(current_milli_time()) + ".opf") - with open(path_tmp_opf, 'w') as fd: - copyfileobj(p.stdout, fd) + if config.config_embed_metadata: + quotes = [3, 5] + tmp_dir = get_temp_dir() + calibredb_binarypath = os.path.join(config.config_binariesdir, SUPPORTED_CALIBRE_BINARIES["calibredb"]) + opf_command = [calibredb_binarypath, 'show_metadata', '--as-opf', str(book_id), + '--with-library', config.config_calibre_dir] + p = process_open(opf_command, quotes) + p.wait() + path_tmp_opf = os.path.join(tmp_dir, "metadata_" + str(current_milli_time()) + ".opf") + with open(path_tmp_opf, 'w') as fd: + copyfileobj(p.stdout, fd) quotes = [1, 2, 4, 6] command = [config.config_converterpath, (file_path + format_old_ext), - (file_path + format_new_ext), '--from-opf', path_tmp_opf] + (file_path + format_new_ext)] + if config.config_embed_metadata: + command.extend('--from-opf', path_tmp_opf) if has_cover: command.extend(['--cover', os.path.join(os.path.dirname(file_path), 'cover.jpg')]) quotes_index = 3 diff --git a/cps/templates/config_edit.html b/cps/templates/config_edit.html index 736062f4..8e37b2e5 100644 --- a/cps/templates/config_edit.html +++ b/cps/templates/config_edit.html @@ -103,6 +103,10 @@
+
+ + +
From 9ef89dbcc383099bbcc5573e97eef73efb181b37 Mon Sep 17 00:00:00 2001 From: Ozzie Isaacs Date: Sun, 10 Dec 2023 07:26:26 +0100 Subject: [PATCH 11/18] Bugfix convert file from gdrive without cover Update Teststatus --- cps/tasks/convert.py | 19 +- test/Calibre-Web TestSummary_Linux.html | 6405 +++++++++++++++++++++-- 2 files changed, 5854 insertions(+), 570 deletions(-) diff --git a/cps/tasks/convert.py b/cps/tasks/convert.py index a39b3f61..913b92dd 100755 --- a/cps/tasks/convert.py +++ b/cps/tasks/convert.py @@ -66,29 +66,32 @@ class TaskConvert(CalibreTask): df = gdriveutils.getFileFromEbooksFolder(cur_book.path, data.name + "." + self.settings['old_book_format'].lower()) df_cover = gdriveutils.getFileFromEbooksFolder(cur_book.path, "cover.jpg") - if df and df_cover: + if df: datafile = os.path.join(config.config_calibre_dir, cur_book.path, data.name + "." + self.settings['old_book_format'].lower()) - datafile_cover = os.path.join(config.config_calibre_dir, - cur_book.path, "cover.jpg") + if df_cover: + datafile_cover = os.path.join(config.config_calibre_dir, + cur_book.path, "cover.jpg") if not os.path.exists(os.path.join(config.config_calibre_dir, cur_book.path)): os.makedirs(os.path.join(config.config_calibre_dir, cur_book.path)) df.GetContentFile(datafile) - df_cover.GetContentFile(datafile_cover) + if df_cover: + df_cover.GetContentFile(datafile_cover) worker_db.session.close() else: - # ToDo Include cover in error handling + # ToDo Include cover in error handling error_message = _("%(format)s not found on Google Drive: %(fn)s", format=self.settings['old_book_format'], fn=data.name + "." + self.settings['old_book_format'].lower()) worker_db.session.close() - return error_message + return self._handleError(self, error_message) filename = self._convert_ebook_format() if config.config_use_google_drive: os.remove(self.file_path + '.' + self.settings['old_book_format'].lower()) - os.remove(os.path.join(config.config_calibre_dir, cur_book.path, "cover.jpg")) + if df_cover: + os.remove(os.path.join(config.config_calibre_dir, cur_book.path, "cover.jpg")) if filename: if config.config_use_google_drive: @@ -254,7 +257,7 @@ class TaskConvert(CalibreTask): command = [config.config_converterpath, (file_path + format_old_ext), (file_path + format_new_ext)] if config.config_embed_metadata: - command.extend('--from-opf', path_tmp_opf) + command.extend(['--from-opf', path_tmp_opf]) if has_cover: command.extend(['--cover', os.path.join(os.path.dirname(file_path), 'cover.jpg')]) quotes_index = 3 diff --git a/test/Calibre-Web TestSummary_Linux.html b/test/Calibre-Web TestSummary_Linux.html index 7ca3dad5..03883e3b 100644 --- a/test/Calibre-Web TestSummary_Linux.html +++ b/test/Calibre-Web TestSummary_Linux.html @@ -37,20 +37,20 @@
-

Start Time: 2023-10-16 19:38:22

+

Start Time: 2023-12-10 09:21:07

-

Stop Time: 2023-10-17 02:18:49

+

Stop Time: 2023-12-10 14:53:07

-

Duration: 5h 37 min

+

Duration: 4h 31 min

@@ -236,13 +236,13 @@ TestBackupMetadata - 22 - 22 + 21 + 21 0 0 0 - Detail + Detail @@ -429,15 +429,6 @@ - -
TestBackupMetadata - test_gdrive
- - PASS - - - - -
TestBackupMetadata - test_upload_book
@@ -861,12 +852,12 @@ - + TestEbookConvertCalibreGDrive 6 - 6 - 0 - 0 + 4 + 1 + 1 0 Detail @@ -875,20 +866,62 @@ - +
TestEbookConvertCalibreGDrive - test_convert_email
- PASS + +
+ ERROR +
+ + + + - +
TestEbookConvertCalibreGDrive - test_convert_failed_and_email
- PASS + +
+ FAIL +
+ + + + @@ -1932,12 +1965,12 @@ - + TestLoadMetadata 1 + 1 0 0 - 1 0 Detail @@ -1946,42 +1979,22 @@ - +
TestLoadMetadata - test_load_metadata
- -
- ERROR -
- - - - + PASS - + TestEditBooksOnGdrive 18 - 17 + 18 + 0 0 - 1 0 Detail @@ -2125,31 +2138,11 @@ IndexError: list index out of range - +
TestEditBooksOnGdrive - test_edit_title
- -
- ERROR -
- - - - + PASS @@ -2691,12 +2684,12 @@ KeyError: 'title' - + TestLdapLogin 13 - 13 - 0 + 11 0 + 2 0 Detail @@ -2804,20 +2797,91 @@ KeyError: 'title' - +
TestLdapLogin - test_ldap_opds_anonymous
- PASS + +
+ ERROR +
+ + + + - +
TestLdapLogin - test_ldap_opds_download_book
- PASS + +
+ ERROR +
+ + + + @@ -3092,15 +3156,15 @@ KeyError: 'title' - + TestLogin + 18 17 - 17 - 0 + 1 0 0 - Detail + Detail @@ -3241,7 +3305,36 @@ KeyError: 'title' - + + +
TestLogin - test_proxy_login_multi_user
+ + +
+ FAIL +
+ + + + + + + + +
TestLogin - test_proxy_login_opds
@@ -3250,7 +3343,7 @@ KeyError: 'title' - +
TestLogin - test_robots
@@ -3623,39 +3716,104 @@ KeyError: 'title' - + TestReader - 6 + 8 5 1 - 0 - 0 + 1 + 1 - Detail + Detail - + + +
TestReader - test_cb7_reader
+ + SKIP + + + + +
TestReader - test_comic_MACOS_files
- PASS + +
+ ERROR +
+ + + + - +
TestReader - test_comic_reader
- PASS + +
+ FAIL +
+ + + + - +
TestReader - test_epub_reader
@@ -3664,7 +3822,7 @@ KeyError: 'title' - +
TestReader - test_pdf_reader
@@ -3673,42 +3831,25 @@ KeyError: 'title' - + -
TestReader - test_sound_listener
+
TestReader - test_single_file_comic
- -
- FAIL -
- - - + PASS + + + + + + +
TestReader - test_sound_listener
+ PASS - +
TestReader - test_txt_reader
@@ -4047,11 +4188,11 @@ AssertionError: '0:02' != '0:01' - - TestThumbnailsEnv - 1 + + TestSocket 1 0 + 1 0 0 @@ -4061,113 +4202,684 @@ AssertionError: '0:02' != '0:01' - + -
TestThumbnailsEnv - test_cover_cache_env_on_database_change
+
TestSocket - test_socket_communication
+ + +
+ FAIL +
+ + + - PASS - - TestThumbnails - 8 - 6 + + TestSystemdActivation 1 0 + 0 1 + 0 - Detail + Detail - + -
TestThumbnails - test_cache_non_writable
+
TestSystemdActivation - test_systemd_activation
+ + +
+ ERROR +
+ + + - PASS + + + + TestThumbnailsEnv + 1 + 0 + 0 + 1 + 0 + + Detail + + + + - + -
TestThumbnails - test_cache_of_deleted_book
+
TestThumbnailsEnv - test_cover_cache_env_on_database_change
+ + +
+ ERROR +
+ + + - PASS + + + + _ErrorHolder + 6 + 0 + 0 + 6 + 0 + + Detail + + + + - + -
TestThumbnails - test_cover_cache_on_database_change
+
tearDownClass (test_thumbnail_env)
+ + +
+ ERROR +
+ + + - PASS - + -
TestThumbnails - test_cover_change_on_upload_new_cover
+
tearDownClass (test_thumbnails)
+ + +
+ ERROR +
+ + + - PASS - + -
TestThumbnails - test_cover_for_series
+
tearDownClass (test_upload_epubs)
+ + +
+ ERROR +
+ + + - SKIP - + -
TestThumbnails - test_cover_on_upload_book
+
tearDownClass (test_user_list)
+ + +
+ ERROR +
+ + + - PASS - + -
TestThumbnails - test_remove_cover_from_cache
+
tearDownClass (test_user_load)
+ + +
+ ERROR +
+ + + - PASS - + -
TestThumbnails - test_sideloaded_book
+
tearDownClass (test_visiblilitys)
- FAIL + ERROR
- @@ -852,12 +852,12 @@ - + TestEbookConvertCalibreGDrive 6 - 4 - 1 - 1 + 6 + 0 + 0 0 Detail @@ -866,62 +866,20 @@ - +
TestEbookConvertCalibreGDrive - test_convert_email
- -
- ERROR -
- - - - + PASS - +
TestEbookConvertCalibreGDrive - test_convert_failed_and_email
- -
- FAIL -
- - - - + PASS @@ -1989,11 +1947,11 @@ AssertionError: 'Finished' != 'Started' - + TestEditBooksOnGdrive 18 - 18 - 0 + 17 + 1 0 0 @@ -2156,11 +2114,31 @@ AssertionError: 'Finished' != 'Started' - +
TestEditBooksOnGdrive - test_watch_metadata
- PASS + +
+ FAIL +
+ + + + @@ -2824,10 +2802,12 @@ urllib3.exceptions.LocationParseError: Failed to parse: http://127.0.0.1:{}/opds During handling of the above exception, another exception occurred: Traceback (most recent call last): - File "/home/ozzie/Development/calibre-web-test/test/test_ldap.py", line 969, in test_ldap_opds_anonymous - r = req_session.get('http://127.0.0.1:{}' + entries['elements'][0]['download']) - File "/home/ozzie/Development/calibre-web-test/venv/lib/python3.10/site-packages/requests/sessions.py", line 602, in get - return self.request("GET", url, **kwargs) + File "/home/ozzie/Development/calibre-web-test/test/test_ldap.py", line 985, in test_ldap_opds_anonymous + r = requests.get('http://127.0.0.1:{}' + entries['elements'][0]['download']) + File "/home/ozzie/Development/calibre-web-test/venv/lib/python3.10/site-packages/requests/api.py", line 73, in get + return request("get", url, params=params, **kwargs) + File "/home/ozzie/Development/calibre-web-test/venv/lib/python3.10/site-packages/requests/api.py", line 59, in request + return session.request(method=method, url=url, **kwargs) File "/home/ozzie/Development/calibre-web-test/venv/lib/python3.10/site-packages/requests/sessions.py", line 575, in request prep = self.prepare_request(req) File "/home/ozzie/Development/calibre-web-test/venv/lib/python3.10/site-packages/requests/sessions.py", line 486, in prepare_request @@ -3043,13 +3023,13 @@ dom.find/</<@chrome://remote/content/shared/DOM.sys.mjs:132:16 - + TestLogging 9 - 8 - 0 + 7 0 1 + 1 Detail @@ -3146,11 +3126,113 @@ dom.find/</<@chrome://remote/content/shared/DOM.sys.mjs:132:16 - +
TestLogging - test_logviewer
- PASS + +
+ ERROR +
+ + + + + + + + + + + _ErrorHolder + 1 + 0 + 0 + 1 + 0 + + Detail + + + + + + + +
tearDownClass (test_logging)
+ + +
+ ERROR +
+ + + + @@ -3164,13 +3246,13 @@ dom.find/</<@chrome://remote/content/shared/DOM.sys.mjs:132:16 0 0 - Detail + Detail - +
TestLogin - test_digest_login
@@ -3179,7 +3261,7 @@ dom.find/</<@chrome://remote/content/shared/DOM.sys.mjs:132:16 - +
TestLogin - test_login_capital_letters_user_unicode_password
@@ -3188,7 +3270,7 @@ dom.find/</<@chrome://remote/content/shared/DOM.sys.mjs:132:16 - +
TestLogin - test_login_cookie_steal
@@ -3197,7 +3279,7 @@ dom.find/</<@chrome://remote/content/shared/DOM.sys.mjs:132:16 - +
TestLogin - test_login_delete_admin
@@ -3206,7 +3288,7 @@ dom.find/</<@chrome://remote/content/shared/DOM.sys.mjs:132:16 - +
TestLogin - test_login_empty_password
@@ -3215,7 +3297,7 @@ dom.find/</<@chrome://remote/content/shared/DOM.sys.mjs:132:16 - +
TestLogin - test_login_locale_select
@@ -3224,7 +3306,7 @@ dom.find/</<@chrome://remote/content/shared/DOM.sys.mjs:132:16 - +
TestLogin - test_login_protected
@@ -3233,7 +3315,7 @@ dom.find/</<@chrome://remote/content/shared/DOM.sys.mjs:132:16 - +
TestLogin - test_login_remember_me
@@ -3242,7 +3324,7 @@ dom.find/</<@chrome://remote/content/shared/DOM.sys.mjs:132:16 - +
TestLogin - test_login_rename_user
@@ -3251,7 +3333,7 @@ dom.find/</<@chrome://remote/content/shared/DOM.sys.mjs:132:16 - +
TestLogin - test_login_unicode_user_space_end_password
@@ -3260,7 +3342,7 @@ dom.find/</<@chrome://remote/content/shared/DOM.sys.mjs:132:16 - +
TestLogin - test_login_user_with_space_password_end_space
@@ -3269,7 +3351,7 @@ dom.find/</<@chrome://remote/content/shared/DOM.sys.mjs:132:16 - +
TestLogin - test_magic_remote_login
@@ -3278,7 +3360,7 @@ dom.find/</<@chrome://remote/content/shared/DOM.sys.mjs:132:16 - +
TestLogin - test_next
@@ -3287,7 +3369,7 @@ dom.find/</<@chrome://remote/content/shared/DOM.sys.mjs:132:16 - +
TestLogin - test_password_policy
@@ -3296,7 +3378,7 @@ dom.find/</<@chrome://remote/content/shared/DOM.sys.mjs:132:16 - +
TestLogin - test_proxy_login
@@ -3305,19 +3387,19 @@ dom.find/</<@chrome://remote/content/shared/DOM.sys.mjs:132:16 - +
TestLogin - test_proxy_login_multi_user
- FAIL + FAIL
-