diff --git a/.gitignore b/.gitignore index 14da8a03..18bd81b5 100644 --- a/.gitignore +++ b/.gitignore @@ -33,3 +33,4 @@ settings.yaml gdrive_credentials client_secrets.json gmail.json +/.key diff --git a/cps/__init__.py b/cps/__init__.py index 9b957090..525cc31b 100644 --- a/cps/__init__.py +++ b/cps/__init__.py @@ -28,6 +28,7 @@ import mimetypes from flask import Flask from .MyLoginManager import MyLoginManager from flask_principal import Principal +from flask_limiter import Limiter from . import logger from .cli import CliParameter @@ -81,7 +82,7 @@ app.config.update( lm = MyLoginManager() -config = config_sql._ConfigSQL() +config = config_sql.ConfigSQL() cli_param = CliParameter() @@ -96,6 +97,7 @@ web_server = WebServer() updater_thread = Updater() +limiter = Limiter(key_func=True, headers_enabled=True) def create_app(): if csrf: @@ -106,7 +108,12 @@ def create_app(): ub.init_db(cli_param.settings_path, cli_param.user_credentials) # pylint: disable=no-member - config_sql.load_configuration(config, ub.session, cli_param) + encrypt_key, error = config_sql.get_encryption_key(os.path.dirname(cli_param.settings_path)) + + config_sql.load_configuration(ub.session, encrypt_key) + config.init_config(ub.session, encrypt_key, cli_param) + if error: + log.error(error) lm.login_view = 'web.login' lm.anonymous_user = ub.Anonymous @@ -150,7 +157,7 @@ def create_app(): if os.environ.get('FLASK_DEBUG'): cache_buster.init_cache_busting(app) log.info('Starting Calibre Web...') - + limiter.init_app(app) Principal(app) lm.init_app(app) app.secret_key = os.getenv('SECRET_KEY', config_sql.get_flask_session_key(ub.session)) @@ -165,7 +172,7 @@ def create_app(): services.ldap.init_app(app, config) if services.goodreads_support: services.goodreads_support.connect(config.config_goodreads_api_key, - config.config_goodreads_api_secret, + config.config_goodreads_api_secret_e, config.config_use_goodreads) config.store_calibre_uuid(calibre_db, db.Library_Id) # Register scheduled tasks diff --git a/cps/admin.py b/cps/admin.py index 27d17a1d..daf7a8e0 100755 --- a/cps/admin.py +++ b/cps/admin.py @@ -206,12 +206,12 @@ def admin(): commit = version['version'] all_user = ub.session.query(ub.User).all() - email_settings = config.get_mail_settings() + # email_settings = mail_config.get_mail_settings() schedule_time = format_time(datetime_time(hour=config.schedule_start_time), format="short") t = timedelta(hours=config.schedule_duration // 60, minutes=config.schedule_duration % 60) schedule_duration = format_timedelta(t, threshold=.99) - return render_title_template("admin.html", allUser=all_user, email=email_settings, config=config, commit=commit, + return render_title_template("admin.html", allUser=all_user, config=config, commit=commit, feature_support=feature_support, schedule_time=schedule_time, schedule_duration=schedule_duration, title=_(u"Admin page"), page="admin") @@ -1062,7 +1062,7 @@ def _config_checkbox_int(to_save, x): def _config_string(to_save, x): - return config.set_from_dictionary(to_save, x, lambda y: y.strip() if y else y) + return config.set_from_dictionary(to_save, x, lambda y: y.strip().strip(u'\u200B\u200C\u200D\ufeff') if y else y) def _configuration_gdrive_helper(to_save): @@ -1151,9 +1151,9 @@ def _configuration_ldap_helper(to_save): reboot_required |= _config_string(to_save, "config_ldap_cert_path") reboot_required |= _config_string(to_save, "config_ldap_key_path") _config_string(to_save, "config_ldap_group_name") - if to_save.get("config_ldap_serv_password", "") != "": + if to_save.get("config_ldap_serv_password_e", "") != "": reboot_required |= 1 - config.set_from_dictionary(to_save, "config_ldap_serv_password", base64.b64encode, encode='UTF-8') + config.set_from_dictionary(to_save, "config_ldap_serv_password_e") config.save() if not config.config_ldap_provider_url \ @@ -1165,7 +1165,7 @@ def _configuration_ldap_helper(to_save): if config.config_ldap_authentication > constants.LDAP_AUTH_ANONYMOUS: if config.config_ldap_authentication > constants.LDAP_AUTH_UNAUTHENTICATE: - if not config.config_ldap_serv_username or not bool(config.config_ldap_serv_password): + if not config.config_ldap_serv_username or not bool(config.config_ldap_serv_password_e): return reboot_required, _configuration_result(_('Please Enter a LDAP Service Account and Password')) else: if not config.config_ldap_serv_username: @@ -1233,7 +1233,7 @@ def new_user(): kobo_support=kobo_support, registered_oauth=oauth_check) -@admi.route("/admin/mailsettings") +@admi.route("/admin/mailsettings", methods=["GET"]) @login_required @admin_required def edit_mailsettings(): @@ -1266,11 +1266,12 @@ def update_mailsettings(): else: _config_int(to_save, "mail_port") _config_int(to_save, "mail_use_ssl") - _config_string(to_save, "mail_password") + _config_string(to_save, "mail_password_e") _config_int(to_save, "mail_size", lambda y: int(y)*1024*1024) - config.mail_server = to_save.get('mail_server', "").strip() - config.mail_from = to_save.get('mail_from', "").strip() - config.mail_login = to_save.get('mail_login', "").strip() + _config_string(to_save, "mail_server") + _config_string(to_save, "mail_from") + _config_string(to_save, "mail_login") + try: config.save() except (OperationalError, InvalidRequestError) as e: @@ -1326,7 +1327,7 @@ def update_scheduledtasks(): error = False to_save = request.form.to_dict() if 0 <= int(to_save.get("schedule_start_time")) <= 23: - _config_int(to_save, "schedule_start_time") + _config_int( to_save, "schedule_start_time") else: flash(_(u"Invalid start time for task specified"), category="error") error = True @@ -1749,10 +1750,10 @@ def _configuration_update_helper(): # Goodreads configuration _config_checkbox(to_save, "config_use_goodreads") _config_string(to_save, "config_goodreads_api_key") - _config_string(to_save, "config_goodreads_api_secret") + _config_string(to_save, "config_goodreads_api_secret_e") if services.goodreads_support: services.goodreads_support.connect(config.config_goodreads_api_key, - config.config_goodreads_api_secret, + config.config_goodreads_api_secret_e, config.config_use_goodreads) _config_int(to_save, "config_updatechannel") diff --git a/cps/config_sql.py b/cps/config_sql.py index 19184202..781e892c 100644 --- a/cps/config_sql.py +++ b/cps/config_sql.py @@ -23,6 +23,10 @@ import json from sqlalchemy import Column, String, Integer, SmallInteger, Boolean, BLOB, JSON from sqlalchemy.exc import OperationalError from sqlalchemy.sql.expression import text +from sqlalchemy import exists +from cryptography.fernet import Fernet +import cryptography.exceptions +from base64 import urlsafe_b64decode try: # Compatibility with sqlalchemy 2.0 from sqlalchemy.orm import declarative_base @@ -56,7 +60,8 @@ class _Settings(_Base): mail_port = Column(Integer, default=25) mail_use_ssl = Column(SmallInteger, default=0) mail_login = Column(String, default='mail@example.com') - mail_password = Column(String, default='mypassword') + mail_password_e = Column(String) + mail_password = Column(String) mail_from = Column(String, default='automailer ') mail_size = Column(Integer, default=25*1024*1024) mail_server_type = Column(SmallInteger, default=0) @@ -106,6 +111,7 @@ class _Settings(_Base): config_use_goodreads = Column(Boolean, default=False) config_goodreads_api_key = Column(String) + config_goodreads_api_secret_e = Column(String) config_goodreads_api_secret = Column(String) config_register_email = Column(Boolean, default=False) config_login_type = Column(Integer, default=0) @@ -116,7 +122,8 @@ class _Settings(_Base): config_ldap_port = Column(SmallInteger, default=389) config_ldap_authentication = Column(SmallInteger, default=constants.LDAP_AUTH_SIMPLE) config_ldap_serv_username = Column(String, default='cn=admin,dc=example,dc=org') - config_ldap_serv_password = Column(String, default="") + config_ldap_serv_password_e = Column(String) + config_ldap_serv_password = Column(String) config_ldap_encryption = Column(SmallInteger, default=0) config_ldap_cacert_path = Column(String, default="") config_ldap_cert_path = Column(String, default="") @@ -159,19 +166,117 @@ class _Settings(_Base): return self.__class__.__name__ +class MailConfigSQL(object): + + def __init__(self): + self.__dict__["dirty"] = list() + + def init_config(self, session, secret_key): + self._session = session + self._settings = None + self._fernet = Fernet(secret_key) + self.load() + + def _read_from_storage(self): + if self._settings is None: + log.debug("_MailConfigSQL._read_from_storage") + self._settings = self._session.query(_Mail_Settings).first() + return self._settings + + def to_dict(self): + storage = {} + for k, v in self.__dict__.items(): + if k[0] != '_' and not k.endswith("password") and not k.endswith("secret") and not k == "cli": + storage[k] = v + return storage + + def load(self): + """Load all configuration values from the underlying storage.""" + s = self._read_from_storage() # type: _Settings + for k, v in s.__dict__.items(): + if k[0] != '_': + if v is None: + # if the storage column has no value, apply the (possible) default + column = s.__class__.__dict__.get(k) + if column.default is not None: + v = column.default.arg + if k.endswith("enc") and v is not None: + try: + setattr(s, k, self._fernet.decrypt(v).decode()) + except cryptography.exceptions.InvalidKey: + setattr(s, k, None) + else: + setattr(self, k, v) + self.__dict__["dirty"] = list() + + def save(self): + """Apply all configuration values to the underlying storage.""" + s = self._read_from_storage() # type: _Settings + for k in self.dirty: + if k[0] == '_': + continue + if hasattr(s, k): + if k.endswith("enc"): + setattr(s, k, self._fernet.encrypt(self.__dict__[k].encode())) + else: + setattr(s, k, self.__dict__[k]) + + log.debug("_MailConfigSQL updating storage") + self._session.merge(s) + try: + self._session.commit() + except OperationalError as e: + log.error('Database error: %s', e) + self._session.rollback() + self.load() + + + def set_from_dictionary(self, dictionary, field, convertor=None, default=None, encode=None): + """Possibly updates a field of this object. + The new value, if present, is grabbed from the given dictionary, and optionally passed through a convertor. + + :returns: `True` if the field has changed value + """ + new_value = dictionary.get(field, default) + if new_value is None: + return False + + if field not in self.__dict__: + log.warning("_ConfigSQL trying to set unknown field '%s' = %r", field, new_value) + return False + + if convertor is not None: + if encode: + new_value = convertor(new_value.encode(encode)) + else: + new_value = convertor(new_value) + + current_value = self.__dict__.get(field) + if current_value == new_value: + return False + + setattr(self, field, new_value) + return True + + def __setattr__(self, attr_name, attr_value): + super().__setattr__(attr_name, attr_value) + self.__dict__["dirty"].append(attr_name) + + # Class holds all application specific settings in calibre-web -class _ConfigSQL(object): +class ConfigSQL(object): # pylint: disable=no-member def __init__(self): - pass + self.__dict__["dirty"] = list() - def init_config(self, session, cli): + def init_config(self, session, secret_key, cli): self._session = session self._settings = None self.db_configured = None self.config_calibre_dir = None - self.load() + self._fernet = Fernet(secret_key) self.cli = cli + self.load() change = False if self.config_converterpath == None: # pylint: disable=access-member-before-definition @@ -300,10 +405,10 @@ class _ConfigSQL(object): setattr(self, field, new_value) return True - def toDict(self): + def to_dict(self): storage = {} for k, v in self.__dict__.items(): - if k[0] != '_' and not k.endswith("password") and not k.endswith("secret") and not k == "cli": + if k[0] != '_' and not k.endswith("_e") and not k == "cli": storage[k] = v return storage @@ -317,7 +422,13 @@ class _ConfigSQL(object): column = s.__class__.__dict__.get(k) if column.default is not None: v = column.default.arg - setattr(self, k, v) + if k.endswith("_e") and v is not None: + try: + setattr(self, k, self._fernet.decrypt(v).decode()) + except cryptography.fernet.InvalidToken: + setattr(self, k, "") + else: + setattr(self, k, v) have_metadata_db = bool(self.config_calibre_dir) if have_metadata_db: @@ -339,16 +450,20 @@ class _ConfigSQL(object): except OperationalError as e: log.error('Database error: %s', e) self._session.rollback() + self.__dict__["dirty"] = list() def save(self): """Apply all configuration values to the underlying storage.""" s = self._read_from_storage() # type: _Settings - for k, v in self.__dict__.items(): + for k in self.dirty: if k[0] == '_': continue if hasattr(s, k): - setattr(s, k, v) + if k.endswith("_e"): + setattr(s, k, self._fernet.encrypt(self.__dict__[k].encode())) + else: + setattr(s, k, self.__dict__[k]) log.debug("_ConfigSQL updating storage") self._session.merge(s) @@ -364,7 +479,6 @@ class _ConfigSQL(object): log.error(error) log.warning("invalidating configuration") self.db_configured = False - # self.config_calibre_dir = None self.save() def store_calibre_uuid(self, calibre_db, Library_table): @@ -376,8 +490,40 @@ class _ConfigSQL(object): except AttributeError: pass + def __setattr__(self, attr_name, attr_value): + super().__setattr__(attr_name, attr_value) + self.__dict__["dirty"].append(attr_name) -def _migrate_table(session, orm_class): + +def _encrypt_fields(session, secret_key): + try: + session.query(exists().where(_Settings.mail_password_e)).scalar() + except OperationalError: + with session.bind.connect() as conn: + conn.execute("ALTER TABLE settings ADD column 'mail_password_e' String") + conn.execute("ALTER TABLE settings ADD column 'config_goodreads_api_secret_e' String") + conn.execute("ALTER TABLE settings ADD column 'config_ldap_serv_password_e' String") + session.commit() + crypter = Fernet(secret_key) + settings = session.query(_Settings.mail_password, _Settings.config_goodreads_api_secret, + _Settings.config_ldap_serv_password).first() + if settings.mail_password: + session.query(_Settings).update( + {_Settings.mail_password_e: crypter.encrypt(settings.mail_password.encode())}) + if settings.config_goodreads_api_secret: + session.query(_Settings).update( + {_Settings.config_goodreads_api_secret_e: + crypter.encrypt(settings.config_goodreads_api_secret.encode())}) + if settings.config_ldap_serv_password: + session.query(_Settings).update( + {_Settings.config_ldap_serv_password_e: + crypter.encrypt(settings.config_ldap_serv_password.encode())}) + session.commit() + + +def _migrate_table(session, orm_class, secret_key=None): + if secret_key: + _encrypt_fields(session, secret_key) changed = False for column_name, column in orm_class.__dict__.items(): @@ -453,22 +599,18 @@ def autodetect_kepubify_binary(): return "" -def _migrate_database(session): +def _migrate_database(session, secret_key): # make sure the table is created, if it does not exist _Base.metadata.create_all(session.bind) - _migrate_table(session, _Settings) + _migrate_table(session, _Settings, secret_key) _migrate_table(session, _Flask_Settings) -def load_configuration(conf, session, cli): - _migrate_database(session) - +def load_configuration(session, secret_key): + _migrate_database(session, secret_key) if not session.query(_Settings).count(): session.add(_Settings()) session.commit() - # conf = _ConfigSQL() - conf.init_config(session, cli) - # return conf def get_flask_session_key(_session): @@ -478,3 +620,25 @@ def get_flask_session_key(_session): _session.add(flask_settings) _session.commit() return flask_settings.flask_session_key + + +def get_encryption_key(key_path): + key_file = os.path.join(key_path, ".key") + generate = True + error = "" + if os.path.exists(key_file) and os.path.getsize(key_file) > 32: + with open(key_file, "rb") as f: + key = f.read() + try: + urlsafe_b64decode(key) + generate = False + except ValueError: + pass + if generate: + key = Fernet.generate_key() + try: + with open(key_file, "wb") as f: + f.write(key) + except PermissionError as e: + error = e + return key, error diff --git a/cps/debug_info.py b/cps/debug_info.py index 6cb30edb..82ca8ca6 100644 --- a/cps/debug_info.py +++ b/cps/debug_info.py @@ -65,7 +65,7 @@ def send_debug(): file_list.remove(element) memory_zip = BytesIO() with zipfile.ZipFile(memory_zip, 'w', compression=zipfile.ZIP_DEFLATED) as zf: - zf.writestr('settings.txt', json.dumps(config.toDict(), sort_keys=True, indent=2)) + zf.writestr('settings.txt', json.dumps(config.to_dict(), sort_keys=True, indent=2)) zf.writestr('libs.txt', json.dumps(collect_stats(), sort_keys=True, indent=2, cls=lazyEncoder)) for fp in file_list: zf.write(fp, os.path.basename(fp)) diff --git a/cps/helper.py b/cps/helper.py index 11da616d..2c1a2127 100755 --- a/cps/helper.py +++ b/cps/helper.py @@ -19,7 +19,6 @@ import os import io -import sys import mimetypes import re import shutil diff --git a/cps/services/simpleldap.py b/cps/services/simpleldap.py index 1ca7e5bf..872538d1 100644 --- a/cps/services/simpleldap.py +++ b/cps/services/simpleldap.py @@ -44,15 +44,15 @@ def init_app(app, config): app.config['LDAP_SCHEMA'] = 'ldap' if config.config_ldap_authentication > constants.LDAP_AUTH_ANONYMOUS: if config.config_ldap_authentication > constants.LDAP_AUTH_UNAUTHENTICATE: - if config.config_ldap_serv_password is None: - config.config_ldap_serv_password = '' - app.config['LDAP_PASSWORD'] = base64.b64decode(config.config_ldap_serv_password) + if config.config_ldap_serv_password_e is None: + config.config_ldap_serv_password_e = '' + app.config['LDAP_PASSWORD'] = config.config_ldap_serv_password_e else: - app.config['LDAP_PASSWORD'] = base64.b64decode("") + app.config['LDAP_PASSWORD'] = "" app.config['LDAP_USERNAME'] = config.config_ldap_serv_username else: app.config['LDAP_USERNAME'] = "" - app.config['LDAP_PASSWORD'] = base64.b64decode("") + app.config['LDAP_PASSWORD'] = "" if bool(config.config_ldap_cert_path): app.config['LDAP_CUSTOM_OPTIONS'].update({ pyLDAP.OPT_X_TLS_REQUIRE_CERT: pyLDAP.OPT_X_TLS_DEMAND, diff --git a/cps/tasks/mail.py b/cps/tasks/mail.py index be240c79..6119ee94 100755 --- a/cps/tasks/mail.py +++ b/cps/tasks/mail.py @@ -202,8 +202,8 @@ class TaskEmail(CalibreTask): self.asyncSMTP.set_debuglevel(1) if use_ssl == 1: self.asyncSMTP.starttls() - if self.settings["mail_password"]: - self.asyncSMTP.login(str(self.settings["mail_login"]), str(self.settings["mail_password"])) + if self.settings["mail_password_e"]: + self.asyncSMTP.login(str(self.settings["mail_login"]), str(self.settings["mail_password_e"])) # Convert message to something to send fp = StringIO() diff --git a/cps/templates/admin.html b/cps/templates/admin.html index 553dcbb6..d2f1c1a6 100755 --- a/cps/templates/admin.html +++ b/cps/templates/admin.html @@ -61,27 +61,27 @@

{{_('E-mail Server Settings')}}

{% if config.get_mail_server_configured() %} - {% if email.mail_server_type == 0 %} + {% if config.mail_server_type == 0 %}
{{_('SMTP Hostname')}}
-
{{email.mail_server}}
+
{{config.mail_server}}
{{_('SMTP Port')}}
-
{{email.mail_port}}
+
{{config.mail_port}}
{{_('Encryption')}}
-
{{ display_bool_setting(email.mail_use_ssl) }}
+
{{ display_bool_setting(config.mail_use_ssl) }}
{{_('SMTP Login')}}
-
{{email.mail_login}}
+
{{config.mail_login}}
{{_('From E-mail')}}
-
{{email.mail_from}}
+
{{config.mail_from}}
{% else %} @@ -92,7 +92,7 @@
{{_('From E-mail')}}
-
{{email.mail_gmail_token['email']}}
+
{{config.mail_gmail_token['email']}}
{% endif %} diff --git a/cps/templates/config_edit.html b/cps/templates/config_edit.html index f6ccb5b3..f6c0d75d 100644 --- a/cps/templates/config_edit.html +++ b/cps/templates/config_edit.html @@ -159,8 +159,8 @@
- - + +
{% endif %} @@ -245,8 +245,8 @@
- - + +
diff --git a/cps/templates/email_edit.html b/cps/templates/email_edit.html index 2a844209..ffdae7ac 100644 --- a/cps/templates/email_edit.html +++ b/cps/templates/email_edit.html @@ -48,8 +48,8 @@
- - + +
diff --git a/optional-requirements.txt b/optional-requirements.txt index 97f355f2..25a07ba6 100644 --- a/optional-requirements.txt +++ b/optional-requirements.txt @@ -33,7 +33,7 @@ scholarly>=1.2.0,<1.7 markdown2>=2.0.0,<2.5.0 html2text>=2020.1.16,<2022.1.1 python-dateutil>=2.1,<2.9.0 -beautifulsoup4>=4.0.1,<4.11.0 +beautifulsoup4>=4.0.1,<4.12.0 cchardet>=2.0.0,<2.2.0 # Comics