Merge branch 'metadata' into Develop

pull/2997/head
Ozzie Isaacs 4 months ago
commit 74c61d9685

@ -49,9 +49,9 @@ sorted_modules = OrderedDict((sorted(modules.items(), key=lambda x: x[0].casefol
def collect_stats():
if constants.NIGHTLY_VERSION[0] == "$Format:%H$":
calibre_web_version = constants.STABLE_VERSION['version']
calibre_web_version = constants.STABLE_VERSION['version'].replace("b", " Beta")
else:
calibre_web_version = (constants.STABLE_VERSION['version'] + ' - '
calibre_web_version = (constants.STABLE_VERSION['version'].replace("b", " Beta") + ' - '
+ constants.NIGHTLY_VERSION[0].replace('%', '%%') + ' - '
+ constants.NIGHTLY_VERSION[1].replace('%', '%%'))

@ -47,7 +47,7 @@ from . import constants, logger, helper, services, cli_param
from . import db, calibre_db, ub, web_server, config, updater_thread, gdriveutils, \
kobo_sync_status, schedule
from .helper import check_valid_domain, send_test_mail, reset_password, generate_password_hash, check_email, \
valid_email, check_username
valid_email, check_username, get_calibre_binarypath
from .gdriveutils import is_gdrive_ready, gdrive_support
from .render_template import render_title_template, get_sidebar_config
from .services.worker import WorkerThread
@ -217,7 +217,7 @@ def admin():
form_date += timedelta(hours=int(commit[20:22]), minutes=int(commit[23:]))
commit = format_datetime(form_date - tz, format='short')
else:
commit = version['version']
commit = version['version'].replace("b", " Beta")
all_user = ub.session.query(ub.User).all()
# email_settings = mail_config.get_mail_settings()
@ -1751,6 +1751,7 @@ def _configuration_update_helper():
_config_checkbox_int(to_save, "config_uploading")
_config_checkbox_int(to_save, "config_unicode_filename")
_config_checkbox_int(to_save, "config_embed_metadata")
# Reboot on config_anonbrowse with enabled ldap, as decoraters are changed in this case
reboot_required |= (_config_checkbox_int(to_save, "config_anonbrowse")
and config.config_login_type == constants.LOGIN_LDAP)
@ -1767,8 +1768,14 @@ def _configuration_update_helper():
constants.EXTENSIONS_UPLOAD = config.config_upload_formats.split(',')
_config_string(to_save, "config_calibre")
_config_string(to_save, "config_converterpath")
_config_string(to_save, "config_binariesdir")
_config_string(to_save, "config_kepubifypath")
if "config_binariesdir" in to_save:
calibre_status = helper.check_calibre(config.config_binariesdir)
if calibre_status:
return _configuration_result(calibre_status)
to_save["config_converterpath"] = get_calibre_binarypath("ebook-convert")
_config_string(to_save, "config_converterpath")
reboot_required |= _config_int(to_save, "config_login_type")

@ -29,8 +29,8 @@ from .constants import DEFAULT_SETTINGS_FILE, DEFAULT_GDRIVE_FILE
def version_info():
if _NIGHTLY_VERSION[1].startswith('$Format'):
return "Calibre-Web version: %s - unknown git-clone" % _STABLE_VERSION['version']
return "Calibre-Web version: %s -%s" % (_STABLE_VERSION['version'], _NIGHTLY_VERSION[1])
return "Calibre-Web version: %s - unknown git-clone" % _STABLE_VERSION['version'].replace("b", " Beta")
return "Calibre-Web version: %s -%s" % (_STABLE_VERSION['version'].replace("b", " Beta"), _NIGHTLY_VERSION[1])
class CliParameter(object):

@ -34,6 +34,7 @@ except ImportError:
from sqlalchemy.ext.declarative import declarative_base
from . import constants, logger
from .subproc_wrapper import process_wait
log = logger.create()
@ -140,10 +141,12 @@ class _Settings(_Base):
config_kepubifypath = Column(String, default=None)
config_converterpath = Column(String, default=None)
config_binariesdir = Column(String, default=None)
config_calibre = Column(String)
config_rarfile_location = Column(String, default=None)
config_upload_formats = Column(String, default=','.join(constants.EXTENSIONS_UPLOAD))
config_unicode_filename = Column(Boolean, default=False)
config_embed_metadata = Column(Boolean, default=True)
config_updatechannel = Column(Integer, default=constants.UPDATE_STABLE)
@ -186,9 +189,11 @@ class ConfigSQL(object):
self.load()
change = False
if self.config_converterpath == None: # pylint: disable=access-member-before-definition
if self.config_binariesdir == None: # pylint: disable=access-member-before-definition
change = True
self.config_converterpath = autodetect_calibre_binary()
self.config_binariesdir = autodetect_calibre_binaries()
self.config_converterpath = autodetect_converter_binary(self.config_binariesdir)
if self.config_kepubifypath == None: # pylint: disable=access-member-before-definition
change = True
@ -474,17 +479,32 @@ def _migrate_table(session, orm_class, secret_key=None):
session.rollback()
def autodetect_calibre_binary():
def autodetect_calibre_binaries():
if sys.platform == "win32":
calibre_path = ["C:\\program files\\calibre\\ebook-convert.exe",
"C:\\program files(x86)\\calibre\\ebook-convert.exe",
"C:\\program files(x86)\\calibre2\\ebook-convert.exe",
"C:\\program files\\calibre2\\ebook-convert.exe"]
calibre_path = ["C:\\program files\\calibre\\",
"C:\\program files(x86)\\calibre\\",
"C:\\program files(x86)\\calibre2\\",
"C:\\program files\\calibre2\\"]
else:
calibre_path = ["/opt/calibre/ebook-convert"]
calibre_path = ["/opt/calibre/"]
for element in calibre_path:
if os.path.isfile(element) and os.access(element, os.X_OK):
return element
supported_binary_paths = [os.path.join(element, binary) for binary in constants.SUPPORTED_CALIBRE_BINARIES.values()]
if all(os.path.isfile(binary_path) and os.access(binary_path, os.X_OK) for binary_path in supported_binary_paths):
values = [process_wait([binary_path, "--version"], pattern='\(calibre (.*)\)') for binary_path in supported_binary_paths]
if all(values):
version = values[0].group(1)
log.debug("calibre version %s", version)
return element
return ""
def autodetect_converter_binary(calibre_path):
if sys.platform == "win32":
converter_path = os.path.join(calibre_path, "ebook-convert.exe")
else:
converter_path = os.path.join(calibre_path, "ebook-convert")
if calibre_path and os.path.isfile(converter_path) and os.access(converter_path, os.X_OK):
return converter_path
return ""
@ -526,6 +546,7 @@ def load_configuration(session, secret_key):
session.commit()
def get_flask_session_key(_session):
flask_settings = _session.query(_Flask_Settings).one_or_none()
if flask_settings == None:

@ -156,6 +156,11 @@ EXTENSIONS_UPLOAD = {'txt', 'pdf', 'epub', 'kepub', 'mobi', 'azw', 'azw3', 'cbr'
'prc', 'doc', 'docx', 'fb2', 'html', 'rtf', 'lit', 'odt', 'mp3', 'mp4', 'ogg',
'opus', 'wav', 'flac', 'm4a', 'm4b'}
_extension = ""
if sys.platform == "win32":
_extension = ".exe"
SUPPORTED_CALIBRE_BINARIES = {binary:binary + _extension for binary in ["ebook-convert", "calibredb"]}
def has_flag(value, bit_flag):
return bit_flag == (bit_flag & (value or 0))
@ -169,13 +174,11 @@ BookMeta = namedtuple('BookMeta', 'file_path, extension, title, author, cover, d
'series_id, languages, publisher, pubdate, identifiers')
# python build process likes to have x.y.zbw -> b for beta and w a counting number
STABLE_VERSION = {'version': '0.6.22 Beta'}
STABLE_VERSION = {'version': '0.6.22b'}
NIGHTLY_VERSION = dict()
NIGHTLY_VERSION[0] = '$Format:%H$'
NIGHTLY_VERSION[1] = '$Format:%cI$'
# NIGHTLY_VERSION[0] = 'bb7d2c6273ae4560e83950d36d64533343623a57'
# NIGHTLY_VERSION[1] = '2018-09-09T10:13:08+02:00'
# CACHE
CACHE_TYPE_THUMBNAILS = 'thumbnails'

@ -23,10 +23,12 @@ from lxml import etree
from . import isoLanguages, cover
from . import config, logger
from .helper import split_authors
from .epub_helper import get_content_opf, default_ns
from .constants import BookMeta
log = logger.create()
def _extract_cover(zip_file, cover_file, cover_path, tmp_file_name):
if cover_file is None:
return None
@ -44,24 +46,14 @@ def _extract_cover(zip_file, cover_file, cover_path, tmp_file_name):
return cover.cover_processing(tmp_file_name, cf, extension)
def get_epub_layout(book, book_data):
ns = {
'n': 'urn:oasis:names:tc:opendocument:xmlns:container',
'pkg': 'http://www.idpf.org/2007/opf',
}
file_path = os.path.normpath(os.path.join(config.get_book_path(),
book.path, book_data.name + "." + book_data.format.lower()))
try:
epubZip = zipfile.ZipFile(file_path)
txt = epubZip.read('META-INF/container.xml')
tree = etree.fromstring(txt)
cfname = tree.xpath('n:rootfiles/n:rootfile/@full-path', namespaces=ns)[0]
cf = epubZip.read(cfname)
tree = etree.fromstring(cf)
p = tree.xpath('/pkg:package/pkg:metadata', namespaces=ns)[0]
tree, __ = get_content_opf(file_path, default_ns)
p = tree.xpath('/pkg:package/pkg:metadata', namespaces=default_ns)[0]
layout = p.xpath('pkg:meta[@property="rendition:layout"]/text()', namespaces=ns)
layout = p.xpath('pkg:meta[@property="rendition:layout"]/text()', namespaces=default_ns)
except (etree.XMLSyntaxError, KeyError, IndexError) as e:
log.error("Could not parse epub metadata of book {} during kobo sync: {}".format(book.id, e))
layout = []
@ -80,12 +72,7 @@ def get_epub_info(tmp_file_path, original_file_name, original_file_extension):
}
epub_zip = zipfile.ZipFile(tmp_file_path)
txt = epub_zip.read('META-INF/container.xml')
tree = etree.fromstring(txt)
cf_name = tree.xpath('n:rootfiles/n:rootfile/@full-path', namespaces=ns)[0]
cf = epub_zip.read(cf_name)
tree = etree.fromstring(cf)
tree, cf_name = get_content_opf(epub_zip, ns)
cover_path = os.path.dirname(cf_name)

@ -0,0 +1,162 @@
# -*- coding: utf-8 -*-
# This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web)
# Copyright (C) 2018 lemmsh, Kennyl, Kyosfonica, matthazinski
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import zipfile
from lxml import etree
from . import isoLanguages
default_ns = {
'n': 'urn:oasis:names:tc:opendocument:xmlns:container',
'pkg': 'http://www.idpf.org/2007/opf',
}
OPF_NAMESPACE = "http://www.idpf.org/2007/opf"
PURL_NAMESPACE = "http://purl.org/dc/elements/1.1/"
OPF = "{%s}" % OPF_NAMESPACE
PURL = "{%s}" % PURL_NAMESPACE
etree.register_namespace("opf", OPF_NAMESPACE)
etree.register_namespace("dc", PURL_NAMESPACE)
OPF_NS = {None: OPF_NAMESPACE} # the default namespace (no prefix)
NSMAP = {'dc': PURL_NAMESPACE, 'opf': OPF_NAMESPACE}
def updateEpub(src, dest, filename, data, ):
# create a temp copy of the archive without filename
with zipfile.ZipFile(src, 'r') as zin:
with zipfile.ZipFile(dest, 'w') as zout:
zout.comment = zin.comment # preserve the comment
for item in zin.infolist():
if item.filename != filename:
zout.writestr(item, zin.read(item.filename))
# now add filename with its new data
with zipfile.ZipFile(dest, mode='a', compression=zipfile.ZIP_DEFLATED) as zf:
zf.writestr(filename, data)
def get_content_opf(file_path, ns=default_ns):
epubZip = zipfile.ZipFile(file_path)
txt = epubZip.read('META-INF/container.xml')
tree = etree.fromstring(txt)
cf_name = tree.xpath('n:rootfiles/n:rootfile/@full-path', namespaces=ns)[0]
cf = epubZip.read(cf_name)
return etree.fromstring(cf), cf_name
def create_new_metadata_backup(book, custom_columns, export_language, translated_cover_name, lang_type=3):
# generate root package element
package = etree.Element(OPF + "package", nsmap=OPF_NS)
package.set("unique-identifier", "uuid_id")
package.set("version", "2.0")
# generate metadata element and all sub elements of it
metadata = etree.SubElement(package, "metadata", nsmap=NSMAP)
identifier = etree.SubElement(metadata, PURL + "identifier", id="calibre_id", nsmap=NSMAP)
identifier.set(OPF + "scheme", "calibre")
identifier.text = str(book.id)
identifier2 = etree.SubElement(metadata, PURL + "identifier", id="uuid_id", nsmap=NSMAP)
identifier2.set(OPF + "scheme", "uuid")
identifier2.text = book.uuid
title = etree.SubElement(metadata, PURL + "title", nsmap=NSMAP)
title.text = book.title
for author in book.authors:
creator = etree.SubElement(metadata, PURL + "creator", nsmap=NSMAP)
creator.text = str(author.name)
creator.set(OPF + "file-as", book.author_sort) # ToDo Check
creator.set(OPF + "role", "aut")
contributor = etree.SubElement(metadata, PURL + "contributor", nsmap=NSMAP)
contributor.text = "calibre (5.7.2) [https://calibre-ebook.com]"
contributor.set(OPF + "file-as", "calibre") # ToDo Check
contributor.set(OPF + "role", "bkp")
date = etree.SubElement(metadata, PURL + "date", nsmap=NSMAP)
date.text = '{d.year:04}-{d.month:02}-{d.day:02}T{d.hour:02}:{d.minute:02}:{d.second:02}'.format(d=book.pubdate)
if book.comments and book.comments[0].text:
for b in book.comments:
description = etree.SubElement(metadata, PURL + "description", nsmap=NSMAP)
description.text = b.text
for b in book.publishers:
publisher = etree.SubElement(metadata, PURL + "publisher", nsmap=NSMAP)
publisher.text = str(b.name)
if not book.languages:
language = etree.SubElement(metadata, PURL + "language", nsmap=NSMAP)
language.text = export_language
else:
for b in book.languages:
language = etree.SubElement(metadata, PURL + "language", nsmap=NSMAP)
language.text = str(b.lang_code) if lang_type == 3 else isoLanguages.get(part3=b.lang_code).part1
for b in book.tags:
subject = etree.SubElement(metadata, PURL + "subject", nsmap=NSMAP)
subject.text = str(b.name)
etree.SubElement(metadata, "meta", name="calibre:author_link_map",
content="{" + ", ".join(['"' + str(a.name) + '": ""' for a in book.authors]) + "}",
nsmap=NSMAP)
for b in book.series:
etree.SubElement(metadata, "meta", name="calibre:series",
content=str(str(b.name)),
nsmap=NSMAP)
if book.series:
etree.SubElement(metadata, "meta", name="calibre:series_index",
content=str(book.series_index),
nsmap=NSMAP)
if len(book.ratings) and book.ratings[0].rating > 0:
etree.SubElement(metadata, "meta", name="calibre:rating",
content=str(book.ratings[0].rating),
nsmap=NSMAP)
etree.SubElement(metadata, "meta", name="calibre:timestamp",
content='{d.year:04}-{d.month:02}-{d.day:02}T{d.hour:02}:{d.minute:02}:{d.second:02}'.format(
d=book.timestamp),
nsmap=NSMAP)
etree.SubElement(metadata, "meta", name="calibre:title_sort",
content=book.sort,
nsmap=NSMAP)
sequence = 0
for cc in custom_columns:
value = None
extra = None
cc_entry = getattr(book, "custom_column_" + str(cc.id))
if cc_entry.__len__():
value = [c.value for c in cc_entry] if cc.is_multiple else cc_entry[0].value
extra = cc_entry[0].extra if hasattr(cc_entry[0], "extra") else None
etree.SubElement(metadata, "meta", name="calibre:user_metadata:#{}".format(cc.label),
content=cc.to_json(value, extra, sequence),
nsmap=NSMAP)
sequence += 1
# generate guide element and all sub elements of it
# Title is translated from default export language
guide = etree.SubElement(package, "guide")
etree.SubElement(guide, "reference", type="cover", title=translated_cover_name, href="cover.jpg")
return package
def replace_metadata(tree, package):
rep_element = tree.xpath('/pkg:package/pkg:metadata', namespaces=default_ns)[0]
new_element = package.xpath('//metadata', namespaces=default_ns)[0]
tree.replace(rep_element, new_element)
return etree.tostring(tree,
xml_declaration=True,
encoding='utf-8',
pretty_print=True).decode('utf-8')

@ -0,0 +1,32 @@
# -*- coding: utf-8 -*-
# This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web)
# Copyright (C) 2023 OzzieIsaacs
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from tempfile import gettempdir
import os
import shutil
def get_temp_dir():
tmp_dir = os.path.join(gettempdir(), 'calibre_web')
if not os.path.isdir(tmp_dir):
os.mkdir(tmp_dir)
return tmp_dir
def del_temp_dir():
tmp_dir = os.path.join(gettempdir(), 'calibre_web')
shutil.rmtree(tmp_dir)

@ -23,7 +23,6 @@
import os
import hashlib
import json
import tempfile
from uuid import uuid4
from time import time
from shutil import move, copyfile
@ -34,6 +33,7 @@ from flask_login import login_required
from . import logger, gdriveutils, config, ub, calibre_db, csrf
from .admin import admin_required
from .file_helper import get_temp_dir
gdrive = Blueprint('gdrive', __name__, url_prefix='/gdrive')
log = logger.create()
@ -139,9 +139,7 @@ try:
dbpath = os.path.join(config.config_calibre_dir, "metadata.db").encode()
if not response['deleted'] and response['file']['title'] == 'metadata.db' \
and response['file']['md5Checksum'] != hashlib.md5(dbpath): # nosec
tmp_dir = os.path.join(tempfile.gettempdir(), 'calibre_web')
if not os.path.isdir(tmp_dir):
os.mkdir(tmp_dir)
tmp_dir = get_temp_dir()
log.info('Database file updated')
copyfile(dbpath, os.path.join(tmp_dir, "metadata.db_" + str(current_milli_time())))

@ -34,7 +34,6 @@ except ImportError:
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.exc import OperationalError, InvalidRequestError, IntegrityError
from sqlalchemy.orm.exc import StaleDataError
from sqlalchemy.sql.expression import text
try:
from httplib2 import __version__ as httplib2_version

@ -25,9 +25,10 @@ import re
import shutil
import socket
from datetime import datetime, timedelta
from tempfile import gettempdir
import requests
import unidecode
from uuid import uuid4
from lxml import etree
from flask import send_from_directory, make_response, redirect, abort, url_for
from flask_babel import gettext as _
@ -54,12 +55,14 @@ from . import calibre_db, cli_param
from .tasks.convert import TaskConvert
from . import logger, config, db, ub, fs
from . import gdriveutils as gd
from .constants import STATIC_DIR as _STATIC_DIR, CACHE_TYPE_THUMBNAILS, THUMBNAIL_TYPE_COVER, THUMBNAIL_TYPE_SERIES
from .subproc_wrapper import process_wait
from .constants import STATIC_DIR as _STATIC_DIR, CACHE_TYPE_THUMBNAILS, THUMBNAIL_TYPE_COVER, THUMBNAIL_TYPE_SERIES, SUPPORTED_CALIBRE_BINARIES
from .subproc_wrapper import process_wait, process_open
from .services.worker import WorkerThread
from .tasks.mail import TaskEmail
from .tasks.thumbnail import TaskClearCoverThumbnailCache, TaskGenerateCoverThumbnails
from .tasks.metadata_backup import TaskBackupMetadata
from .file_helper import get_temp_dir
from .epub_helper import get_content_opf, create_new_metadata_backup, updateEpub, replace_metadata
log = logger.create()
@ -921,10 +924,7 @@ def save_cover(img, book_path):
return False, _("Only jpg/jpeg files are supported as coverfile")
if config.config_use_google_drive:
tmp_dir = os.path.join(gettempdir(), 'calibre_web')
if not os.path.isdir(tmp_dir):
os.mkdir(tmp_dir)
tmp_dir = get_temp_dir()
ret, message = save_cover_from_filestorage(tmp_dir, "uploaded_cover.jpg", img)
if ret is True:
gd.uploadFileToEbooksFolder(os.path.join(book_path, 'cover.jpg').replace("\\", "/"),
@ -938,29 +938,87 @@ def save_cover(img, book_path):
def do_download_file(book, book_format, client, data, headers):
book_name = data.name
if config.config_use_google_drive:
# startTime = time.time()
df = gd.getFileFromEbooksFolder(book.path, data.name + "." + book_format)
df = gd.getFileFromEbooksFolder(book.path, book_name + "." + book_format)
# log.debug('%s', time.time() - startTime)
if df:
return gd.do_gdrive_download(df, headers)
if config.config_embed_metadata and (
(book_format == "kepub" and config.config_kepubifypath ) or
(book_format != "kepub" and config.config_binariesdir)):
output_path = os.path.join(config.config_calibre_dir, book.path)
if not os.path.exists(output_path):
os.makedirs(output_path)
output = os.path.join(config.config_calibre_dir, book.path, book_name + "." + book_format)
gd.downloadFile(book.path, book_name + "." + book_format, output)
if book_format == "kepub" and config.config_kepubifypath:
filename, download_name = do_kepubify_metadata_replace(book, output)
elif book_format != "kepub" and config.config_binariesdir:
filename, download_name = do_calibre_export(book.id, book_format)
else:
return gd.do_gdrive_download(df, headers)
else:
abort(404)
else:
filename = os.path.join(config.get_book_path(), book.path)
if not os.path.isfile(os.path.join(filename, data.name + "." + book_format)):
if not os.path.isfile(os.path.join(filename, book_name + "." + book_format)):
# ToDo: improve error handling
log.error('File not found: %s', os.path.join(filename, data.name + "." + book_format))
log.error('File not found: %s', os.path.join(filename, book_name + "." + book_format))
if client == "kobo" and book_format == "kepub":
headers["Content-Disposition"] = headers["Content-Disposition"].replace(".kepub", ".kepub.epub")
response = make_response(send_from_directory(filename, data.name + "." + book_format))
# ToDo Check headers parameter
for element in headers:
response.headers[element[0]] = element[1]
log.info('Downloading file: {}'.format(os.path.join(filename, data.name + "." + book_format)))
return response
if book_format == "kepub" and config.config_kepubifypath and config.config_embed_metadata:
filename, download_name = do_kepubify_metadata_replace(book, os.path.join(filename,
book_name + "." + book_format))
elif book_format != "kepub" and config.config_binariesdir and config.config_embed_metadata:
filename, download_name = do_calibre_export(book.id, book_format)
else:
download_name = book_name
response = make_response(send_from_directory(filename, download_name + "." + book_format))
# ToDo Check headers parameter
for element in headers:
response.headers[element[0]] = element[1]
log.info('Downloading file: {}'.format(os.path.join(filename, book_name + "." + book_format)))
return response
def do_kepubify_metadata_replace(book, file_path):
custom_columns = (calibre_db.session.query(db.CustomColumns)
.filter(db.CustomColumns.mark_for_delete == 0)
.filter(db.CustomColumns.datatype.notin_(db.cc_exceptions))
.order_by(db.CustomColumns.label).all())
tree, cf_name = get_content_opf(file_path)
package = create_new_metadata_backup(book, custom_columns, current_user.locale, _("Cover"), lang_type=2)
content = replace_metadata(tree, package)
tmp_dir = get_temp_dir()
temp_file_name = str(uuid4())
# open zipfile and replace metadata block in content.opf
updateEpub(file_path, os.path.join(tmp_dir, temp_file_name + ".kepub"), cf_name, content)
return tmp_dir, temp_file_name
def do_calibre_export(book_id, book_format, ):
try:
quotes = [3, 5, 7, 9]
tmp_dir = get_temp_dir()
calibredb_binarypath = get_calibre_binarypath("calibredb")
temp_file_name = str(uuid4())
opf_command = [calibredb_binarypath, 'export', '--dont-write-opf', '--with-library', config.config_calibre_dir,
'--to-dir', tmp_dir, '--formats', book_format, "--template", "{}".format(temp_file_name),
str(book_id)]
p = process_open(opf_command, quotes)
_, err = p.communicate()
if err:
log.error('Metadata embedder encountered an error: %s', err)
return tmp_dir, temp_file_name
except OSError as ex:
# ToDo real error handling
log.error_or_exception(ex)
##################################
@ -984,6 +1042,47 @@ def check_unrar(unrar_location):
return _('Error executing UnRar')
def check_calibre(calibre_location):
if not calibre_location:
return
if not os.path.exists(calibre_location):
return _('Could not find the specified directory')
if not os.path.isdir(calibre_location):
return _('Please specify a directory, not a file')
try:
supported_binary_paths = [os.path.join(calibre_location, binary)
for binary in SUPPORTED_CALIBRE_BINARIES.values()]
binaries_available = [os.path.isfile(binary_path) for binary_path in supported_binary_paths]
binaries_executable = [os.access(binary_path, os.X_OK) for binary_path in supported_binary_paths]
if all(binaries_available) and all(binaries_executable):
values = [process_wait([binary_path, "--version"], pattern='\(calibre (.*)\)')
for binary_path in supported_binary_paths]
if all(values):
version = values[0].group(1)
log.debug("calibre version %s", version)
else:
return _('Calibre binaries not viable')
else:
ret_val = []
missing_binaries=[path for path, available in
zip(SUPPORTED_CALIBRE_BINARIES.values(), binaries_available) if not available]
missing_perms=[path for path, available in
zip(SUPPORTED_CALIBRE_BINARIES.values(), binaries_executable) if not available]
if missing_binaries:
ret_val.append(_('Missing calibre binaries: %(missing)s', missing=", ".join(missing_binaries)))
if missing_perms:
ret_val.append(_('Missing executable permissions: %(missing)s', missing=", ".join(missing_perms)))
return ", ".join(ret_val)
except (OSError, UnicodeDecodeError) as err:
log.error_or_exception(err)
return _('Error excecuting Calibre')
def json_serial(obj):
"""JSON serializer for objects not serializable by default json code"""
@ -1008,7 +1107,7 @@ def tags_filters():
# checks if domain is in database (including wildcards)
# example SELECT * FROM @TABLE WHERE 'abcdefg' LIKE Name;
# example SELECT * FROM @TABLE WHERE 'abcdefg' LIKE Name;
# from https://code.luasoftware.com/tutorials/flask/execute-raw-sql-in-flask-sqlalchemy/
# in all calls the email address is checked for validity
def check_valid_domain(domain_text):
@ -1042,6 +1141,17 @@ def get_download_link(book_id, book_format, client):
abort(404)
def get_calibre_binarypath(binary):
binariesdir = config.config_binariesdir
if binariesdir:
try:
return os.path.join(binariesdir, SUPPORTED_CALIBRE_BINARIES[binary])
except KeyError as ex:
log.error("Binary not supported by Calibre-Web: %s", SUPPORTED_CALIBRE_BINARIES[binary])
pass
return ""
def clear_cover_thumbnail_cache(book_id):
if config.schedule_generate_book_covers:
WorkerThread.add(None, TaskClearCoverThumbnailCache(book_id), hidden=True)

@ -21,6 +21,7 @@ import datetime
from . import config, constants
from .services.background_scheduler import BackgroundScheduler, CronTrigger, use_APScheduler
from .tasks.database import TaskReconnectDatabase
from .tasks.tempFolder import TaskDeleteTempFolder
from .tasks.thumbnail import TaskGenerateCoverThumbnails, TaskGenerateSeriesThumbnails, TaskClearCoverThumbnailCache
from .services.worker import WorkerThread
from .tasks.metadata_backup import TaskBackupMetadata
@ -31,6 +32,9 @@ def get_scheduled_tasks(reconnect=True):
if reconnect:
tasks.append([lambda: TaskReconnectDatabase(), 'reconnect', False])
# Delete temp folder
tasks.append([lambda: TaskDeleteTempFolder(), 'delete temp', True])
# Generate metadata.opf file for each changed book
if config.schedule_metadata_backup:
tasks.append([lambda: TaskBackupMetadata("en"), 'backup metadata', False])
@ -86,6 +90,8 @@ def register_startup_tasks():
# Ignore tasks that should currently be running, as these will be added when registering scheduled tasks
if constants.APP_MODE in ['development', 'test'] and not should_task_be_running(start, duration):
scheduler.schedule_tasks_immediately(tasks=get_scheduled_tasks(False))
else:
scheduler.schedule_tasks_immediately(tasks=[[lambda: TaskDeleteTempFolder(), 'delete temp', True]])
def should_task_be_running(start, duration):

@ -19,8 +19,10 @@
import os
import re
from glob import glob
from shutil import copyfile
from shutil import copyfile, copyfileobj
from markupsafe import escape
from time import time
from uuid import uuid4
from sqlalchemy.exc import SQLAlchemyError
from flask_babel import lazy_gettext as N_
@ -32,13 +34,15 @@ from cps.subproc_wrapper import process_open
from flask_babel import gettext as _
from cps.kobo_sync_status import remove_synced_book
from cps.ub import init_db_thread
from cps.file_helper import get_temp_dir
from cps.tasks.mail import TaskEmail
from cps import gdriveutils
from cps import gdriveutils, helper
from cps.constants import SUPPORTED_CALIBRE_BINARIES
log = logger.create()
current_milli_time = lambda: int(round(time() * 1000))
class TaskConvert(CalibreTask):
def __init__(self, file_path, book_id, task_message, settings, ereader_mail, user=None):
@ -61,24 +65,33 @@ class TaskConvert(CalibreTask):
data = worker_db.get_book_format(self.book_id, self.settings['old_book_format'])
df = gdriveutils.getFileFromEbooksFolder(cur_book.path,
data.name + "." + self.settings['old_book_format'].lower())
df_cover = gdriveutils.getFileFromEbooksFolder(cur_book.path, "cover.jpg")
if df:
datafile = os.path.join(config.get_book_path(),
cur_book.path,
data.name + "." + self.settings['old_book_format'].lower())
if df_cover:
datafile_cover = os.path.join(config.get_book_path(),
cur_book.path, "cover.jpg")
if not os.path.exists(os.path.join(config.get_book_path(), cur_book.path)):
os.makedirs(os.path.join(config.get_book_path(), cur_book.path))
df.GetContentFile(datafile)
if df_cover:
df_cover.GetContentFile(datafile_cover)
worker_db.session.close()
else:
# ToDo Include cover in error handling
error_message = _("%(format)s not found on Google Drive: %(fn)s",
format=self.settings['old_book_format'],
fn=data.name + "." + self.settings['old_book_format'].lower())
worker_db.session.close()
return error_message
return self._handleError(self, error_message)
filename = self._convert_ebook_format()
if config.config_use_google_drive:
os.remove(self.file_path + '.' + self.settings['old_book_format'].lower())
if df_cover:
os.remove(os.path.join(config.config_calibre_dir, cur_book.path, "cover.jpg"))
if filename:
if config.config_use_google_drive:
@ -112,7 +125,7 @@ class TaskConvert(CalibreTask):
# check to see if destination format already exists - or if book is in database
# if it does - mark the conversion task as complete and return a success
# this will allow send to E-Reader workflow to continue to work
# this will allow to send to E-Reader workflow to continue to work
if os.path.isfile(file_path + format_new_ext) or\
local_db.get_book_format(self.book_id, self.settings['new_book_format']):
log.info("Book id %d already converted to %s", book_id, format_new_ext)
@ -152,7 +165,8 @@ class TaskConvert(CalibreTask):
if not os.path.exists(config.config_converterpath):
self._handleError(N_("Calibre ebook-convert %(tool)s not found", tool=config.config_converterpath))
return
check, error_message = self._convert_calibre(file_path, format_old_ext, format_new_ext)
has_cover = local_db.get_book(book_id).has_cover
check, error_message = self._convert_calibre(file_path, format_old_ext, format_new_ext, has_cover)
if check == 0:
cur_book = local_db.get_book(book_id)
@ -194,8 +208,15 @@ class TaskConvert(CalibreTask):
return
def _convert_kepubify(self, file_path, format_old_ext, format_new_ext):
if config.config_embed_metadata:
tmp_dir, temp_file_name = helper.do_calibre_export(self.book_id, format_old_ext[1:])
filename = os.path.join(tmp_dir, temp_file_name + format_old_ext)
temp_file_path = tmp_dir
else:
filename = file_path + format_old_ext
temp_file_path = os.path.dirname(file_path)
quotes = [1, 3]
command = [config.config_kepubifypath, (file_path + format_old_ext), '-o', os.path.dirname(file_path)]
command = [config.config_kepubifypath, filename, '-o', temp_file_path, '-i']
try:
p = process_open(command, quotes)
except OSError as e:
@ -209,13 +230,12 @@ class TaskConvert(CalibreTask):
if p.poll() is not None:
break
# ToD Handle
# process returncode
check = p.returncode
# move file
if check == 0:
converted_file = glob(os.path.join(os.path.dirname(file_path), "*.kepub.epub"))
converted_file = glob(os.path.splitext(filename)[0] + "*.kepub.epub")
if len(converted_file) == 1:
copyfile(converted_file[0], (file_path + format_new_ext))
os.unlink(converted_file[0])
@ -224,16 +244,28 @@ class TaskConvert(CalibreTask):
folder=os.path.dirname(file_path))
return check, None
def _convert_calibre(self, file_path, format_old_ext, format_new_ext):
def _convert_calibre(self, file_path, format_old_ext, format_new_ext, has_cover):
try:
# Linux py2.7 encode as list without quotes no empty element for parameters
# linux py3.x no encode and as list without quotes no empty element for parameters
# windows py2.7 encode as string with quotes empty element for parameters is okay
# windows py 3.x no encode and as string with quotes empty element for parameters is okay
# separate handling for windows and linux
quotes = [1, 2]
# path_tmp_opf = self._embed_metadata()
if config.config_embed_metadata:
quotes = [3, 5]
tmp_dir = get_temp_dir()
calibredb_binarypath = os.path.join(config.config_binariesdir, SUPPORTED_CALIBRE_BINARIES["calibredb"])
opf_command = [calibredb_binarypath, 'show_metadata', '--as-opf', str(self.book_id),
'--with-library', config.config_calibre_dir]
p = process_open(opf_command, quotes)
p.wait()
path_tmp_opf = os.path.join(tmp_dir, "metadata_" + str(uuid4()) + ".opf")
with open(path_tmp_opf, 'w') as fd:
copyfileobj(p.stdout, fd)
quotes = [1, 2, 4, 6]
command = [config.config_converterpath, (file_path + format_old_ext),
(file_path + format_new_ext)]
if config.config_embed_metadata:
command.extend(['--from-opf', path_tmp_opf])
if has_cover:
command.extend(['--cover', os.path.join(os.path.dirname(file_path), 'cover.jpg')])
quotes_index = 3
if config.config_calibre:
parameters = config.config_calibre.split(" ")

@ -17,26 +17,13 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import os
from urllib.request import urlopen
from lxml import etree
from cps import config, db, gdriveutils, logger
from cps.services.worker import CalibreTask
from flask_babel import lazy_gettext as N_
OPF_NAMESPACE = "http://www.idpf.org/2007/opf"
PURL_NAMESPACE = "http://purl.org/dc/elements/1.1/"
OPF = "{%s}" % OPF_NAMESPACE
PURL = "{%s}" % PURL_NAMESPACE
etree.register_namespace("opf", OPF_NAMESPACE)
etree.register_namespace("dc", PURL_NAMESPACE)
OPF_NS = {None: OPF_NAMESPACE} # the default namespace (no prefix)
NSMAP = {'dc': PURL_NAMESPACE, 'opf': OPF_NAMESPACE}
from ..epub_helper import create_new_metadata_backup
class TaskBackupMetadata(CalibreTask):
@ -101,7 +88,8 @@ class TaskBackupMetadata(CalibreTask):
self.calibre_db.session.close()
def open_metadata(self, book, custom_columns):
package = self.create_new_metadata_backup(book, custom_columns)
# package = self.create_new_metadata_backup(book, custom_columns)
package = create_new_metadata_backup(book, custom_columns, self.export_language)
if config.config_use_google_drive:
if not gdriveutils.is_gdrive_ready():
raise Exception('Google Drive is configured but not ready')
@ -123,7 +111,7 @@ class TaskBackupMetadata(CalibreTask):
except Exception as ex:
raise Exception('Writing Metadata failed with error: {} '.format(ex))
def create_new_metadata_backup(self, book, custom_columns):
'''def create_new_metadata_backup(self, book, custom_columns):
# generate root package element
package = etree.Element(OPF + "package", nsmap=OPF_NS)
package.set("unique-identifier", "uuid_id")
@ -208,7 +196,7 @@ class TaskBackupMetadata(CalibreTask):
guide = etree.SubElement(package, "guide")
etree.SubElement(guide, "reference", type="cover", title=self.translated_title, href="cover.jpg")
return package
return package'''
@property
def name(self):

@ -0,0 +1,47 @@
# -*- coding: utf-8 -*-
# This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web)
# Copyright (C) 2023 OzzieIsaacs
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from urllib.request import urlopen
from flask_babel import lazy_gettext as N_
from cps import logger, file_helper
from cps.services.worker import CalibreTask
class TaskDeleteTempFolder(CalibreTask):
def __init__(self, task_message=N_('Delete temp folder contents')):
super(TaskDeleteTempFolder, self).__init__(task_message)
self.log = logger.create()
def run(self, worker_thread):
try:
file_helper.del_temp_dir()
except FileNotFoundError:
pass
except (PermissionError, OSError) as e:
self.log.error("Error deleting temp folder: {}".format(e))
self._handleSuccess()
@property
def name(self):
return "Delete Temp Folder"
@property
def is_cancellable(self):
return False

@ -103,6 +103,10 @@
<input type="checkbox" id="config_unicode_filename" name="config_unicode_filename" {% if config.config_unicode_filename %}checked{% endif %}>
<label for="config_unicode_filename">{{_('Convert non-English characters in title and author while saving to disk')}}</label>
</div>
<div class="form-group">
<input type="checkbox" id="config_embed_metadata" name="config_embed_metadata" {% if config.config_embed_metadata %}checked{% endif %}>
<label for="config_embed_metadata">{{_('Embed Metadata to Ebook File on Download and Conversion (needs Calibre/Kepubify binaries)')}}</label>
</div>
<div class="form-group">
<input type="checkbox" id="config_uploading" data-control="upload_settings" name="config_uploading" {% if config.config_uploading %}checked{% endif %}>
<label for="config_uploading">{{_('Enable Uploads')}} {{_('(Please ensure that users also have upload permissions)')}}</label>
@ -323,12 +327,12 @@
</div>
<div id="collapsefive" class="panel-collapse collapse">
<div class="panel-body">
<label for="config_converterpath">{{_('Path to Calibre E-Book Converter')}}</label>
<label for="config_binariesdir">{{_('Path to Calibre Binaries')}}</label>
<div class="form-group input-group">
<input type="text" class="form-control" id="config_converterpath" name="config_converterpath" value="{% if config.config_converterpath != None %}{{ config.config_converterpath }}{% endif %}" autocomplete="off">
<span class="input-group-btn">
<button type="button" data-toggle="modal" id="converter_modal_path" data-link="config_converterpath" data-target="#fileModal" class="btn btn-default"><span class="glyphicon glyphicon-folder-open"></span></button>
</span>
<input type="text" class="form-control" id="config_binariesdir" name="config_binariesdir" value="{% if config.config_binariesdir != None %}{{ config.config_binariesdir }}{% endif %}" autocomplete="off">
<span class="input-group-btn">
<button type="button" data-toggle="modal" id="binaries_modal_path" data-link="config_binariesdir" data-folderonly="true" data-target="#fileModal" class="btn btn-default"><span class="glyphicon glyphicon-folder-open"></span></button>
</span>
</div>
<div class="form-group">
<label for="config_calibre">{{_('Calibre E-Book Converter Settings')}}</label>

@ -25,13 +25,13 @@ import threading
import time
import zipfile
from io import BytesIO
from tempfile import gettempdir
import requests
from flask_babel import format_datetime
from flask_babel import gettext as _
from . import constants, logger # config, web_server
from .file_helper import get_temp_dir
log = logger.create()
@ -85,7 +85,7 @@ class Updater(threading.Thread):
z = zipfile.ZipFile(BytesIO(r.content))
self.status = 3
log.debug('Extracting zipfile')
tmp_dir = gettempdir()
tmp_dir = get_temp_dir()
z.extractall(tmp_dir)
folder_name = os.path.join(tmp_dir, z.namelist()[0])[:-1]
if not os.path.isdir(folder_name):
@ -566,7 +566,7 @@ class Updater(threading.Thread):
try:
current_version[2] = int(current_version[2])
except ValueError:
current_version[2] = int(current_version[2].split(' ')[0])-1
current_version[2] = int(current_version[2].replace("b", "").split(' ')[0])-1
# Check if major versions are identical search for newest non-equal commit and update to this one
if major_version_update == current_version[0]:

@ -18,12 +18,12 @@
import os
import hashlib
from tempfile import gettempdir
from flask_babel import gettext as _
from . import logger, comic, isoLanguages
from .constants import BookMeta
from .helper import split_authors
from .file_helper import get_temp_dir
log = logger.create()
@ -249,10 +249,7 @@ def get_magick_version():
def upload(uploadfile, rar_excecutable):
tmp_dir = os.path.join(gettempdir(), 'calibre_web')
if not os.path.isdir(tmp_dir):
os.mkdir(tmp_dir)
tmp_dir = get_temp_dir()
filename = uploadfile.filename
filename_root, file_extension = os.path.splitext(filename)

@ -66,7 +66,7 @@ include = cps/services*
[options.extras_require]
gdrive =
google-api-python-client>=1.7.11,<2.98.0
google-api-python-client>=1.7.11,<2.108.0
gevent>20.6.0,<24.0.0
greenlet>=0.4.17,<2.1.0
httplib2>=0.9.2,<0.23.0
@ -79,7 +79,7 @@ gdrive =
rsa>=3.4.2,<4.10.0
gmail =
google-auth-oauthlib>=0.4.3,<1.1.0
google-api-python-client>=1.7.11,<2.98.0
google-api-python-client>=1.7.11,<2.108.0
goodreads =
goodreads>=0.3.2,<0.4.0
python-Levenshtein>=0.12.0,<0.22.0

File diff suppressed because it is too large Load Diff
Loading…
Cancel
Save