From 777c2726d31f2668275150dafbcccbb5afc876ce Mon Sep 17 00:00:00 2001 From: Ozzieisaacs Date: Mon, 7 Dec 2020 08:44:49 +0100 Subject: [PATCH 1/7] Changed session_handing --- cps/__init__.py | 5 ++- cps/admin.py | 112 +++++++++++++++++++++++----------------------- cps/config_sql.py | 8 +++- cps/db.py | 5 ++- cps/editbooks.py | 8 ++-- cps/helper.py | 15 +++---- cps/kobo.py | 51 ++++++++++----------- cps/kobo_auth.py | 12 ++--- cps/oauth_bb.py | 48 ++++++++++---------- cps/opds.py | 12 ++--- cps/shelf.py | 92 ++++++++++++++++++------------------- cps/ub.py | 31 ++++++++----- cps/web.py | 98 ++++++++++++++++++++-------------------- 13 files changed, 254 insertions(+), 243 deletions(-) diff --git a/cps/__init__.py b/cps/__init__.py index 03945b57..575a5a8a 100644 --- a/cps/__init__.py +++ b/cps/__init__.py @@ -71,7 +71,7 @@ lm.session_protection = 'strong' ub.init_db(cli.settingspath) # pylint: disable=no-member -config = config_sql.load_configuration(ub.session) +config = config_sql.load_configuration(ub.Scoped_Session) web_server = WebServer() @@ -94,12 +94,13 @@ def create_app(): app.root_path = app.root_path.decode('utf-8') app.instance_path = app.instance_path.decode('utf-8') + #if os.environ.get('FLASK_DEBUG'): cache_buster.init_cache_busting(app) log.info('Starting Calibre Web...') Principal(app) lm.init_app(app) - app.secret_key = os.getenv('SECRET_KEY', config_sql.get_flask_session_key(ub.session)) + app.secret_key = os.getenv('SECRET_KEY', config_sql.get_flask_session_key(ub.Scoped_Session)) web_server.init_app(app, config) diff --git a/cps/admin.py b/cps/admin.py index b38c1313..f0a82e77 100644 --- a/cps/admin.py +++ b/cps/admin.py @@ -30,7 +30,7 @@ from datetime import datetime, timedelta from babel import Locale as LC from babel.dates import format_datetime -from flask import Blueprint, flash, redirect, url_for, abort, request, make_response, send_from_directory +from flask import Blueprint, flash, redirect, url_for, abort, request, make_response, send_from_directory, g from flask_login import login_required, current_user, logout_user from flask_babel import gettext as _ from sqlalchemy import and_ @@ -88,7 +88,7 @@ def shutdown(): if task in (0, 1): # valid commandos received # close all database connections calibre_db.dispose() - ub.dispose() + # ub.dispose() if task == 0: showtext['text'] = _(u'Server restarted, please reload page') @@ -130,7 +130,7 @@ def admin(): else: commit = version['version'] - allUser = ub.session.query(ub.User).all() + allUser = g.ubsession.query(ub.User).all() email_settings = config.get_mail_settings() kobo_support = feature_support['kobo'] and config.config_kobo_sync return render_title_template("admin.html", allUser=allUser, email=email_settings, config=config, commit=commit, @@ -204,9 +204,9 @@ def edit_domain(allow): # pk: 1 //primary key (record id) # value: 'superuser!' //new value vals = request.form.to_dict() - answer = ub.session.query(ub.Registration).filter(ub.Registration.id == vals['pk']).first() + answer = g.ubsession.query(ub.Registration).filter(ub.Registration.id == vals['pk']).first() answer.domain = vals['value'].replace('*', '%').replace('?', '_').lower() - ub.session.commit() + g.ubsession.commit() return "" @@ -215,12 +215,12 @@ def edit_domain(allow): @admin_required def add_domain(allow): domain_name = request.form.to_dict()['domainname'].replace('*', '%').replace('?', '_').lower() - check = ub.session.query(ub.Registration).filter(ub.Registration.domain == domain_name)\ + check = g.ubsession.query(ub.Registration).filter(ub.Registration.domain == domain_name)\ .filter(ub.Registration.allow == allow).first() if not check: new_domain = ub.Registration(domain=domain_name, allow=allow) - ub.session.add(new_domain) - ub.session.commit() + g.ubsession.add(new_domain) + g.ubsession.commit() return "" @@ -229,13 +229,13 @@ def add_domain(allow): @admin_required def delete_domain(): domain_id = request.form.to_dict()['domainid'].replace('*', '%').replace('?', '_').lower() - ub.session.query(ub.Registration).filter(ub.Registration.id == domain_id).delete() - ub.session.commit() + g.ubsession.query(ub.Registration).filter(ub.Registration.id == domain_id).delete() + g.ubsession.commit() # If last domain was deleted, add all domains by default - if not ub.session.query(ub.Registration).filter(ub.Registration.allow==1).count(): + if not g.ubsession.query(ub.Registration).filter(ub.Registration.allow==1).count(): new_domain = ub.Registration(domain="%.%",allow=1) - ub.session.add(new_domain) - ub.session.commit() + g.ubsession.add(new_domain) + g.ubsession.commit() return "" @@ -243,7 +243,7 @@ def delete_domain(): @login_required @admin_required def list_domain(allow): - answer = ub.session.query(ub.Registration).filter(ub.Registration.allow == allow).all() + answer = g.ubsession.query(ub.Registration).filter(ub.Registration.allow == allow).all() json_dumps = json.dumps([{"domain": r.domain.replace('%', '*').replace('_', '?'), "id": r.id} for r in answer]) js = json.dumps(json_dumps.replace('"', "'")).lstrip('"').strip('"') response = make_response(js.replace("'", '"')) @@ -269,23 +269,23 @@ def edit_restriction(res_type): if res_type == 2: # Tags per user usr_id = os.path.split(request.referrer)[-1] if usr_id.isdigit() == True: - usr = ub.session.query(ub.User).filter(ub.User.id == int(usr_id)).first() + usr = g.ubsession.query(ub.User).filter(ub.User.id == int(usr_id)).first() else: usr = current_user elementlist = usr.list_allowed_tags() elementlist[int(element['id'][1:])]=element['Element'] usr.allowed_tags = ','.join(elementlist) - ub.session.commit() + g.ubsession.commit() if res_type == 3: # CColumn per user usr_id = os.path.split(request.referrer)[-1] if usr_id.isdigit() == True: - usr = ub.session.query(ub.User).filter(ub.User.id == int(usr_id)).first() + usr = g.ubsession.query(ub.User).filter(ub.User.id == int(usr_id)).first() else: usr = current_user elementlist = usr.list_allowed_column_values() elementlist[int(element['id'][1:])]=element['Element'] usr.allowed_column_value = ','.join(elementlist) - ub.session.commit() + g.ubsession.commit() if element['id'].startswith('d'): if res_type == 0: # Tags as template elementlist = config.list_denied_tags() @@ -300,23 +300,23 @@ def edit_restriction(res_type): if res_type == 2: # Tags per user usr_id = os.path.split(request.referrer)[-1] if usr_id.isdigit() == True: - usr = ub.session.query(ub.User).filter(ub.User.id == int(usr_id)).first() + usr = g.ubsession.query(ub.User).filter(ub.User.id == int(usr_id)).first() else: usr = current_user elementlist = usr.list_denied_tags() elementlist[int(element['id'][1:])]=element['Element'] usr.denied_tags = ','.join(elementlist) - ub.session.commit() + g.ubsession.commit() if res_type == 3: # CColumn per user usr_id = os.path.split(request.referrer)[-1] if usr_id.isdigit() == True: - usr = ub.session.query(ub.User).filter(ub.User.id == int(usr_id)).first() + usr = g.ubsession.query(ub.User).filter(ub.User.id == int(usr_id)).first() else: usr = current_user elementlist = usr.list_denied_column_values() elementlist[int(element['id'][1:])]=element['Element'] usr.denied_column_value = ','.join(elementlist) - ub.session.commit() + g.ubsession.commit() return "" def restriction_addition(element, list_func): @@ -357,27 +357,27 @@ def add_restriction(res_type): if res_type == 2: # Tags per user usr_id = os.path.split(request.referrer)[-1] if usr_id.isdigit() == True: - usr = ub.session.query(ub.User).filter(ub.User.id == int(usr_id)).first() + usr = g.ubsession.query(ub.User).filter(ub.User.id == int(usr_id)).first() else: usr = current_user if 'submit_allow' in element: usr.allowed_tags = restriction_addition(element, usr.list_allowed_tags) - ub.session.commit() + g.ubsession.commit() elif 'submit_deny' in element: usr.denied_tags = restriction_addition(element, usr.list_denied_tags) - ub.session.commit() + g.ubsession.commit() if res_type == 3: # CustomC per user usr_id = os.path.split(request.referrer)[-1] if usr_id.isdigit() == True: - usr = ub.session.query(ub.User).filter(ub.User.id == int(usr_id)).first() + usr = g.ubsession.query(ub.User).filter(ub.User.id == int(usr_id)).first() else: usr = current_user if 'submit_allow' in element: usr.allowed_column_value = restriction_addition(element, usr.list_allowed_column_values) - ub.session.commit() + g.ubsession.commit() elif 'submit_deny' in element: usr.denied_column_value = restriction_addition(element, usr.list_denied_column_values) - ub.session.commit() + g.ubsession.commit() return "" @admi.route("/ajax/deleterestriction/", methods=['POST']) @@ -402,27 +402,27 @@ def delete_restriction(res_type): elif res_type == 2: # Tags per user usr_id = os.path.split(request.referrer)[-1] if usr_id.isdigit() == True: - usr = ub.session.query(ub.User).filter(ub.User.id == int(usr_id)).first() + usr = g.ubsession.query(ub.User).filter(ub.User.id == int(usr_id)).first() else: usr = current_user if element['id'].startswith('a'): usr.allowed_tags = restriction_deletion(element, usr.list_allowed_tags) - ub.session.commit() + g.ubsession.commit() elif element['id'].startswith('d'): usr.denied_tags = restriction_deletion(element, usr.list_denied_tags) - ub.session.commit() + g.ubsession.commit() elif res_type == 3: # Columns per user usr_id = os.path.split(request.referrer)[-1] if usr_id.isdigit() == True: # select current user if admins are editing their own rights - usr = ub.session.query(ub.User).filter(ub.User.id == int(usr_id)).first() + usr = g.ubsession.query(ub.User).filter(ub.User.id == int(usr_id)).first() else: usr = current_user if element['id'].startswith('a'): usr.allowed_column_value = restriction_deletion(element, usr.list_allowed_column_values) - ub.session.commit() + g.ubsession.commit() elif element['id'].startswith('d'): usr.denied_column_value = restriction_deletion(element, usr.list_denied_column_values) - ub.session.commit() + g.ubsession.commit() return "" @@ -445,7 +445,7 @@ def list_restriction(res_type): elif res_type == 2: # Tags per user usr_id = os.path.split(request.referrer)[-1] if usr_id.isdigit() == True: - usr = ub.session.query(ub.User).filter(ub.User.id == usr_id).first() + usr = g.ubsession.query(ub.User).filter(ub.User.id == usr_id).first() else: usr = current_user restrict = [{'Element': x, 'type':_('Deny'), 'id': 'd'+str(i) } @@ -456,7 +456,7 @@ def list_restriction(res_type): elif res_type == 3: # CustomC per user usr_id = os.path.split(request.referrer)[-1] if usr_id.isdigit() == True: - usr = ub.session.query(ub.User).filter(ub.User.id==usr_id).first() + usr = g.ubsession.query(ub.User).filter(ub.User.id==usr_id).first() else: usr = current_user restrict = [{'Element': x, 'type':_('Deny'), 'id': 'd'+str(i) } @@ -535,7 +535,7 @@ def _configuration_oauth_helper(to_save): element["active"] = 1 else: element["active"] = 0 - ub.session.query(ub.OAuthProvider).filter(ub.OAuthProvider.id == element['id']).update( + g.ubsession.query(ub.OAuthProvider).filter(ub.OAuthProvider.id == element['id']).update( {"oauth_client_id": to_save["config_" + str(element['id']) + "_oauth_client_id"], "oauth_client_secret": to_save["config_" + str(element['id']) + "_oauth_client_secret"], "active": element["active"]}) @@ -691,7 +691,7 @@ def _configuration_update_helper(): _config_checkbox(to_save, "config_remote_login") if not config.config_remote_login: - ub.session.query(ub.RemoteAuthToken).filter(ub.RemoteAuthToken.token_type==0).delete() + g.ubsession.query(ub.RemoteAuthToken).filter(ub.RemoteAuthToken.token_type==0).delete() # Goodreads configuration _config_checkbox(to_save, "config_use_goodreads") @@ -723,7 +723,7 @@ def _configuration_update_helper(): if unrar_status: return _configuration_result(unrar_status, gdriveError) except (OperationalError, InvalidRequestError): - ub.session.rollback() + g.ubsession.rollback() _configuration_result(_(u"Settings DB is not Writeable"), gdriveError) try: @@ -791,9 +791,9 @@ def _handle_new_user(to_save, content,languages, translations, kobo_support): registered_oauth=oauth_check, kobo_support=kobo_support, title=_(u"Add new user")) content.password = generate_password_hash(to_save["password"]) - existing_user = ub.session.query(ub.User).filter(func.lower(ub.User.nickname) == to_save["nickname"].lower()) \ + existing_user = g.ubsession.query(ub.User).filter(func.lower(ub.User.nickname) == to_save["nickname"].lower()) \ .first() - existing_email = ub.session.query(ub.User).filter(ub.User.email == to_save["email"].lower()) \ + existing_email = g.ubsession.query(ub.User).filter(ub.User.email == to_save["email"].lower()) \ .first() if not existing_user and not existing_email: content.nickname = to_save["nickname"] @@ -814,31 +814,31 @@ def _handle_new_user(to_save, content,languages, translations, kobo_support): content.denied_tags = config.config_denied_tags content.allowed_column_value = config.config_allowed_column_value content.denied_column_value = config.config_denied_column_value - ub.session.add(content) - ub.session.commit() + g.ubsession.add(content) + g.ubsession.commit() flash(_(u"User '%(user)s' created", user=content.nickname), category="success") return redirect(url_for('admin.admin')) except IntegrityError: - ub.session.rollback() + g.ubsession.rollback() flash(_(u"Found an existing account for this e-mail address or nickname."), category="error") except OperationalError: - ub.session.rollback() + g.ubsession.rollback() flash(_(u"Settings DB is not Writeable"), category="error") def _handle_edit_user(to_save, content,languages, translations, kobo_support): if "delete" in to_save: - if ub.session.query(ub.User).filter(ub.User.role.op('&')(constants.ROLE_ADMIN) == constants.ROLE_ADMIN, + if g.ubsession.query(ub.User).filter(ub.User.role.op('&')(constants.ROLE_ADMIN) == constants.ROLE_ADMIN, ub.User.id != content.id).count(): - ub.session.query(ub.User).filter(ub.User.id == content.id).delete() - ub.session.commit() + g.ubsession.query(ub.User).filter(ub.User.id == content.id).delete() + g.ubsession.commit() flash(_(u"User '%(nick)s' deleted", nick=content.nickname), category="success") return redirect(url_for('admin.admin')) else: flash(_(u"No admin user remaining, can't delete user", nick=content.nickname), category="error") return redirect(url_for('admin.admin')) else: - if not ub.session.query(ub.User).filter(ub.User.role.op('&')(constants.ROLE_ADMIN) == constants.ROLE_ADMIN, + if not g.ubsession.query(ub.User).filter(ub.User.role.op('&')(constants.ROLE_ADMIN) == constants.ROLE_ADMIN, ub.User.id != content.id).count() and \ not 'admin_role' in to_save: flash(_(u"No admin user remaining, can't remove admin role", nick=content.nickname), category="error") @@ -872,7 +872,7 @@ def _handle_edit_user(to_save, content,languages, translations, kobo_support): if "locale" in to_save and to_save["locale"]: content.locale = to_save["locale"] if to_save["email"] and to_save["email"] != content.email: - existing_email = ub.session.query(ub.User).filter(ub.User.email == to_save["email"].lower()) \ + existing_email = g.ubsession.query(ub.User).filter(ub.User.email == to_save["email"].lower()) \ .first() if not existing_email: content.email = to_save["email"] @@ -889,7 +889,7 @@ def _handle_edit_user(to_save, content,languages, translations, kobo_support): title=_(u"Edit User %(nick)s", nick=content.nickname), page="edituser") if "nickname" in to_save and to_save["nickname"] != content.nickname: # Query User nickname, if not existing, change - if not ub.session.query(ub.User).filter(ub.User.nickname == to_save["nickname"]).scalar(): + if not g.ubsession.query(ub.User).filter(ub.User.nickname == to_save["nickname"]).scalar(): content.nickname = to_save["nickname"] else: flash(_(u"This username is already taken"), category="error") @@ -906,13 +906,13 @@ def _handle_edit_user(to_save, content,languages, translations, kobo_support): if "kindle_mail" in to_save and to_save["kindle_mail"] != content.kindle_mail: content.kindle_mail = to_save["kindle_mail"] try: - ub.session.commit() + g.ubsession.commit() flash(_(u"User '%(nick)s' updated", nick=content.nickname), category="success") except IntegrityError: - ub.session.rollback() + g.ubsession.rollback() flash(_(u"An unknown error occured."), category="error") except OperationalError: - ub.session.rollback() + g.ubsession.rollback() flash(_(u"Settings DB is not Writeable"), category="error") @@ -961,7 +961,7 @@ def update_mailsettings(): try: config.save() except (OperationalError, InvalidRequestError): - ub.session.rollback() + g.ubsession.rollback() flash(_(u"Settings DB is not Writeable"), category="error") return edit_mailsettings() @@ -985,7 +985,7 @@ def update_mailsettings(): @login_required @admin_required def edit_user(user_id): - content = ub.session.query(ub.User).filter(ub.User.id == int(user_id)).first() # type: ub.User + content = g.ubsession.query(ub.User).filter(ub.User.id == int(user_id)).first() # type: ub.User if not content or (not config.config_anonbrowse and content.nickname == "Guest"): flash(_(u"User not found"), category="error") return redirect(url_for('admin.admin')) diff --git a/cps/config_sql.py b/cps/config_sql.py index 877ad1c2..eabe4ba7 100644 --- a/cps/config_sql.py +++ b/cps/config_sql.py @@ -395,7 +395,8 @@ def _migrate_database(session): _migrate_table(session, _Flask_Settings) -def load_configuration(session): +def load_configuration(Session): + session = Session() _migrate_database(session) if not session.query(_Settings).count(): @@ -409,12 +410,15 @@ def load_configuration(session): session.query(ub.User).filter(ub.User.mature_content != True). \ update({"denied_tags": conf.config_mature_content_tags}, synchronize_session=False) session.commit() + session.close() return conf -def get_flask_session_key(session): +def get_flask_session_key(Session): + session = Session() flask_settings = session.query(_Flask_Settings).one_or_none() if flask_settings == None: flask_settings = _Flask_Settings(os.urandom(32)) session.add(flask_settings) session.commit() + session.close() return flask_settings.flask_session_key diff --git a/cps/db.py b/cps/db.py index f648a794..520d724b 100644 --- a/cps/db.py +++ b/cps/db.py @@ -32,9 +32,10 @@ from sqlalchemy.orm import relationship, sessionmaker, scoped_session from sqlalchemy.orm.collections import InstrumentedList from sqlalchemy.ext.declarative import declarative_base, DeclarativeMeta from sqlalchemy.pool import StaticPool -from flask_login import current_user from sqlalchemy.sql.expression import and_, true, false, text, func, or_ from sqlalchemy.ext.associationproxy import association_proxy +from flask_login import current_user +from flask import g from babel import Locale as LC from babel.core import UnknownLocaleError from flask_babel import gettext as _ @@ -564,7 +565,7 @@ class CalibreDB(): def common_filters(self, allow_show_archived=False): if not allow_show_archived: archived_books = ( - ub.session.query(ub.ArchivedBook) + g.ubsession.query(ub.ArchivedBook) .filter(ub.ArchivedBook.user_id == int(current_user.id)) .filter(ub.ArchivedBook.is_archived == True) .all() diff --git a/cps/editbooks.py b/cps/editbooks.py index 7d207ed3..3f60a4a6 100644 --- a/cps/editbooks.py +++ b/cps/editbooks.py @@ -27,7 +27,7 @@ import json from shutil import copyfile from uuid import uuid4 -from flask import Blueprint, request, flash, redirect, url_for, abort, Markup, Response +from flask import Blueprint, request, flash, redirect, url_for, abort, Markup, Response, g from flask_babel import gettext as _ from flask_login import current_user, login_required from sqlalchemy.exc import OperationalError @@ -212,10 +212,10 @@ def delete_book(book_id, book_format, jsonResponse): flash(error, category="warning") if not book_format: # delete book from Shelfs, Downloads, Read list - ub.session.query(ub.BookShelf).filter(ub.BookShelf.book_id == book_id).delete() - ub.session.query(ub.ReadBook).filter(ub.ReadBook.book_id == book_id).delete() + g.ubsession.query(ub.BookShelf).filter(ub.BookShelf.book_id == book_id).delete() + g.ubsession.query(ub.ReadBook).filter(ub.ReadBook.book_id == book_id).delete() ub.delete_download(book_id) - ub.session.commit() + g.ubsession.commit() # check if only this book links to: # author, language, series, tags, custom columns diff --git a/cps/helper.py b/cps/helper.py index 9845df97..9c725735 100644 --- a/cps/helper.py +++ b/cps/helper.py @@ -24,10 +24,7 @@ import io import mimetypes import re import shutil -import glob import time -import zipfile -import json import unicodedata from datetime import datetime, timedelta from tempfile import gettempdir @@ -35,7 +32,7 @@ from tempfile import gettempdir import requests from babel.dates import format_datetime from babel.units import format_unit -from flask import send_from_directory, make_response, redirect, abort, url_for, send_file +from flask import send_from_directory, make_response, redirect, abort, url_for, g from flask_babel import gettext as _ from flask_login import current_user from sqlalchemy.sql.expression import true, false, and_, text @@ -485,7 +482,7 @@ def delete_book_gdrive(book, book_format): def reset_password(user_id): - existing_user = ub.session.query(ub.User).filter(ub.User.id == user_id).first() + existing_user = g.ubsession.query(ub.User).filter(ub.User.id == user_id).first() if not existing_user: return 0, None if not config.get_mail_server_configured(): @@ -493,11 +490,11 @@ def reset_password(user_id): try: password = generate_random_password() existing_user.password = generate_password_hash(password) - ub.session.commit() + g.ubsession.commit() send_registration_mail(existing_user.email, existing_user.nickname, password, True) return 1, existing_user.nickname except Exception: - ub.session.rollback() + g.ubsession.rollback() return 0, None @@ -779,11 +776,11 @@ def tags_filters(): def check_valid_domain(domain_text): # domain_text = domain_text.split('@', 1)[-1].lower() sql = "SELECT * FROM registration WHERE (:domain LIKE domain and allow = 1);" - result = ub.session.query(ub.Registration).from_statement(text(sql)).params(domain=domain_text).all() + result = g.ubsession.query(ub.Registration).from_statement(text(sql)).params(domain=domain_text).all() if not len(result): return False sql = "SELECT * FROM registration WHERE (:domain LIKE domain and allow = 0);" - result = ub.session.query(ub.Registration).from_statement(text(sql)).params(domain=domain_text).all() + result = g.ubsession.query(ub.Registration).from_statement(text(sql)).params(domain=domain_text).all() return not len(result) diff --git a/cps/kobo.py b/cps/kobo.py index 90d0182e..47d10417 100644 --- a/cps/kobo.py +++ b/cps/kobo.py @@ -37,7 +37,8 @@ from flask import ( current_app, url_for, redirect, - abort + abort, + g ) from flask_login import current_user from werkzeug.datastructures import Headers @@ -210,7 +211,7 @@ def HandleSyncRequest(): # generate reading state data changed_reading_states = ( - ub.session.query(ub.KoboReadingState) + g.ubsession.query(ub.KoboReadingState) .filter(and_(func.datetime(ub.KoboReadingState.last_modified) > sync_token.reading_state_last_modified, ub.KoboReadingState.user_id == current_user.id, ub.KoboReadingState.book_id.notin_(reading_states_in_new_entitlements)))) @@ -439,19 +440,19 @@ def HandleTagCreate(): log.debug("Received malformed v1/library/tags request.") abort(400, description="Malformed tags POST request. Data has empty 'Name', missing 'Name' or 'Items' field") - shelf = ub.session.query(ub.Shelf).filter(ub.Shelf.name == name, ub.Shelf.user_id == + shelf = g.ubsession.query(ub.Shelf).filter(ub.Shelf.name == name, ub.Shelf.user_id == current_user.id).one_or_none() if shelf and not shelf_lib.check_shelf_edit_permissions(shelf): abort(401, description="User is unauthaurized to create shelf.") if not shelf: shelf = ub.Shelf(user_id=current_user.id, name=name, uuid=str(uuid.uuid4())) - ub.session.add(shelf) + g.ubsession.add(shelf) items_unknown_to_calibre = add_items_to_shelf(items, shelf) if items_unknown_to_calibre: log.debug("Received request to add unknown books to a collection. Silently ignoring items.") - ub.session.commit() + g.ubsession.commit() return make_response(jsonify(str(shelf.uuid)), 201) @@ -459,7 +460,7 @@ def HandleTagCreate(): @kobo.route("/v1/library/tags/", methods=["DELETE", "PUT"]) @requires_kobo_auth def HandleTagUpdate(tag_id): - shelf = ub.session.query(ub.Shelf).filter(ub.Shelf.uuid == tag_id, + shelf = g.ubsession.query(ub.Shelf).filter(ub.Shelf.uuid == tag_id, ub.Shelf.user_id == current_user.id).one_or_none() if not shelf: log.debug("Received Kobo tag update request on a collection unknown to CalibreWeb") @@ -483,8 +484,8 @@ def HandleTagUpdate(tag_id): abort(400, description="Malformed tags POST request. Data is missing 'Name' field") shelf.name = name - ub.session.merge(shelf) - ub.session.commit() + g.ubsession.merge(shelf) + g.ubsession.commit() return make_response(' ', 200) @@ -522,7 +523,7 @@ def HandleTagAddItem(tag_id): log.debug("Received malformed v1/library/tags//items/delete request.") abort(400, description="Malformed tags POST request. Data is missing 'Items' field") - shelf = ub.session.query(ub.Shelf).filter(ub.Shelf.uuid == tag_id, + shelf = g.ubsession.query(ub.Shelf).filter(ub.Shelf.uuid == tag_id, ub.Shelf.user_id == current_user.id).one_or_none() if not shelf: log.debug("Received Kobo request on a collection unknown to CalibreWeb") @@ -535,8 +536,8 @@ def HandleTagAddItem(tag_id): if items_unknown_to_calibre: log.debug("Received request to add an unknown book to a collection. Silently ignoring item.") - ub.session.merge(shelf) - ub.session.commit() + g.ubsession.merge(shelf) + g.ubsession.commit() return make_response('', 201) @@ -552,7 +553,7 @@ def HandleTagRemoveItem(tag_id): log.debug("Received malformed v1/library/tags//items/delete request.") abort(400, description="Malformed tags POST request. Data is missing 'Items' field") - shelf = ub.session.query(ub.Shelf).filter(ub.Shelf.uuid == tag_id, + shelf = g.ubsession.query(ub.Shelf).filter(ub.Shelf.uuid == tag_id, ub.Shelf.user_id == current_user.id).one_or_none() if not shelf: log.debug( @@ -577,7 +578,7 @@ def HandleTagRemoveItem(tag_id): shelf.books.filter(ub.BookShelf.book_id == book.id).delete() except KeyError: items_unknown_to_calibre.append(item) - ub.session.commit() + g.ubsession.commit() if items_unknown_to_calibre: log.debug("Received request to remove an unknown book to a collecition. Silently ignoring item.") @@ -590,7 +591,7 @@ def HandleTagRemoveItem(tag_id): def sync_shelves(sync_token, sync_results): new_tags_last_modified = sync_token.tags_last_modified - for shelf in ub.session.query(ub.ShelfArchive).filter(func.datetime(ub.ShelfArchive.last_modified) > sync_token.tags_last_modified, + for shelf in g.ubsession.query(ub.ShelfArchive).filter(func.datetime(ub.ShelfArchive.last_modified) > sync_token.tags_last_modified, ub.ShelfArchive.user_id == current_user.id): new_tags_last_modified = max(shelf.last_modified, new_tags_last_modified) @@ -603,7 +604,7 @@ def sync_shelves(sync_token, sync_results): } }) - for shelf in ub.session.query(ub.Shelf).filter(func.datetime(ub.Shelf.last_modified) > sync_token.tags_last_modified, + for shelf in g.ubsession.query(ub.Shelf).filter(func.datetime(ub.Shelf.last_modified) > sync_token.tags_last_modified, ub.Shelf.user_id == current_user.id): if not shelf_lib.check_shelf_view_permissions(shelf): continue @@ -623,7 +624,7 @@ def sync_shelves(sync_token, sync_results): "ChangedTag": tag }) sync_token.tags_last_modified = new_tags_last_modified - ub.session.commit() + g.ubsession.commit() # Creates a Kobo "Tag" object from a ub.Shelf object @@ -700,11 +701,11 @@ def HandleStateRequest(book_uuid): update_results_response["StatusInfoResult"] = {"Result": "Success"} except (KeyError, TypeError, ValueError, StatementError): log.debug("Received malformed v1/library//state request.") - ub.session.rollback() + g.ubsession.rollback() abort(400, description="Malformed request data is missing 'ReadingStates' key") - ub.session.merge(kobo_reading_state) - ub.session.commit() + g.ubsession.merge(kobo_reading_state) + g.ubsession.commit() return jsonify({ "RequestResult": "Success", "UpdateResults": [update_results_response], @@ -732,7 +733,7 @@ def get_ub_read_status(kobo_read_status): def get_or_create_reading_state(book_id): - book_read = ub.session.query(ub.ReadBook).filter(ub.ReadBook.book_id == book_id, + book_read = g.ubsession.query(ub.ReadBook).filter(ub.ReadBook.book_id == book_id, ub.ReadBook.user_id == current_user.id).one_or_none() if not book_read: book_read = ub.ReadBook(user_id=current_user.id, book_id=book_id) @@ -741,8 +742,8 @@ def get_or_create_reading_state(book_id): kobo_reading_state.current_bookmark = ub.KoboBookmark() kobo_reading_state.statistics = ub.KoboStatistics() book_read.kobo_reading_state = kobo_reading_state - ub.session.add(book_read) - ub.session.commit() + g.ubsession.add(book_read) + g.ubsession.commit() return book_read.kobo_reading_state @@ -835,7 +836,7 @@ def HandleBookDeletionRequest(book_uuid): book_id = book.id archived_book = ( - ub.session.query(ub.ArchivedBook) + g.ubsession.query(ub.ArchivedBook) .filter(ub.ArchivedBook.book_id == book_id) .first() ) @@ -844,8 +845,8 @@ def HandleBookDeletionRequest(book_uuid): archived_book.is_archived = True archived_book.last_modified = datetime.datetime.utcnow() - ub.session.merge(archived_book) - ub.session.commit() + g.ubsession.merge(archived_book) + g.ubsession.commit() return ("", 204) diff --git a/cps/kobo_auth.py b/cps/kobo_auth.py index 0f6cd174..e936acce 100644 --- a/cps/kobo_auth.py +++ b/cps/kobo_auth.py @@ -102,7 +102,7 @@ def requires_kobo_auth(f): auth_token = get_auth_token() if auth_token is not None: user = ( - ub.session.query(ub.User) + g.ubsession.query(ub.User) .join(ub.RemoteAuthToken) .filter(ub.RemoteAuthToken.auth_token == auth_token).filter(ub.RemoteAuthToken.token_type==1) .first() @@ -135,7 +135,7 @@ def generate_auth_token(user_id): ) else: # Invalidate any prevously generated Kobo Auth token for this user. - auth_token = ub.session.query(ub.RemoteAuthToken).filter( + auth_token = g.ubsession.query(ub.RemoteAuthToken).filter( ub.RemoteAuthToken.user_id == user_id ).filter(ub.RemoteAuthToken.token_type==1).first() @@ -146,8 +146,8 @@ def generate_auth_token(user_id): auth_token.auth_token = (hexlify(urandom(16))).decode("utf-8") auth_token.token_type = 1 - ub.session.add(auth_token) - ub.session.commit() + g.ubsession.add(auth_token) + g.ubsession.commit() return render_title_template( "generate_kobo_auth_url.html", title=_(u"Kobo Setup"), @@ -162,7 +162,7 @@ def generate_auth_token(user_id): @login_required def delete_auth_token(user_id): # Invalidate any prevously generated Kobo Auth token for this user. - ub.session.query(ub.RemoteAuthToken).filter(ub.RemoteAuthToken.user_id == user_id)\ + g.ubsession.query(ub.RemoteAuthToken).filter(ub.RemoteAuthToken.user_id == user_id)\ .filter(ub.RemoteAuthToken.token_type==1).delete() - ub.session.commit() + g.ubsession.commit() return "" diff --git a/cps/oauth_bb.py b/cps/oauth_bb.py index 4d489cdd..1305c977 100644 --- a/cps/oauth_bb.py +++ b/cps/oauth_bb.py @@ -24,7 +24,7 @@ from __future__ import division, print_function, unicode_literals import json from functools import wraps -from flask import session, request, make_response, abort +from flask import session, request, make_response, abort, g from flask import Blueprint, flash, redirect, url_for from flask_babel import gettext as _ from flask_dance.consumer import oauth_authorized, oauth_error @@ -74,7 +74,7 @@ def register_user_with_oauth(user=None): else: for oauth_key in all_oauth.keys(): # Find this OAuth token in the database, or create it - query = ub.session.query(ub.OAuth).filter_by( + query = g.ubsession.query(ub.OAuth).filter_by( provider=oauth_key, provider_user_id=session[str(oauth_key) + "_oauth_user_id"], ) @@ -85,10 +85,10 @@ def register_user_with_oauth(user=None): # no found, return error return try: - ub.session.commit() + g.ubsession.commit() except Exception as e: log.exception(e) - ub.session.rollback() + g.ubsession.rollback() def logout_oauth_user(): @@ -99,19 +99,19 @@ def logout_oauth_user(): if ub.oauth_support: oauthblueprints = [] - if not ub.session.query(ub.OAuthProvider).count(): + if not g.ubsession.query(ub.OAuthProvider).count(): oauthProvider = ub.OAuthProvider() oauthProvider.provider_name = "github" oauthProvider.active = False - ub.session.add(oauthProvider) - ub.session.commit() + g.ubsession.add(oauthProvider) + g.ubsession.commit() oauthProvider = ub.OAuthProvider() oauthProvider.provider_name = "google" oauthProvider.active = False - ub.session.add(oauthProvider) - ub.session.commit() + g.ubsession.add(oauthProvider) + g.ubsession.commit() - oauth_ids = ub.session.query(ub.OAuthProvider).all() + oauth_ids = g.ubsession.query(ub.OAuthProvider).all() ele1 = dict(provider_name='github', id=oauth_ids[0].id, active=oauth_ids[0].active, @@ -141,7 +141,7 @@ if ub.oauth_support: scope=element['scope'] ) element['blueprint'] = blueprint - element['blueprint'].backend = OAuthBackend(ub.OAuth, ub.session, str(element['id']), + element['blueprint'].backend = OAuthBackend(ub.OAuth, g.ubsession, str(element['id']), user=current_user, user_required=True) app.register_blueprint(blueprint, url_prefix="/login") if element['active']: @@ -185,7 +185,7 @@ if ub.oauth_support: session[provider_id + "_oauth_token"] = token # Find this OAuth token in the database, or create it - query = ub.session.query(ub.OAuth).filter_by( + query = g.ubsession.query(ub.OAuth).filter_by( provider=provider_id, provider_user_id=provider_user_id, ) @@ -200,11 +200,11 @@ if ub.oauth_support: token=token, ) try: - ub.session.add(oauth_entry) - ub.session.commit() + g.ubsession.add(oauth_entry) + g.ubsession.commit() except Exception as e: log.exception(e) - ub.session.rollback() + g.ubsession.rollback() # Disable Flask-Dance's default behavior for saving the OAuth token # Value differrs depending on flask-dance version @@ -212,7 +212,7 @@ if ub.oauth_support: def bind_oauth_or_register(provider_id, provider_user_id, redirect_url, provider_name): - query = ub.session.query(ub.OAuth).filter_by( + query = g.ubsession.query(ub.OAuth).filter_by( provider=provider_id, provider_user_id=provider_user_id, ) @@ -230,13 +230,13 @@ if ub.oauth_support: if current_user and current_user.is_authenticated: oauth_entry.user = current_user try: - ub.session.add(oauth_entry) - ub.session.commit() + g.ubsession.add(oauth_entry) + g.ubsession.commit() flash(_(u"Link to %(oauth)s Succeeded", oauth=provider_name), category="success") return redirect(url_for('web.profile')) except Exception as e: log.exception(e) - ub.session.rollback() + g.ubsession.rollback() else: flash(_(u"Login failed, No User Linked With OAuth Account"), category="error") log.info('Login failed, No User Linked With OAuth Account') @@ -253,7 +253,7 @@ if ub.oauth_support: def get_oauth_status(): status = [] - query = ub.session.query(ub.OAuth).filter_by( + query = g.ubsession.query(ub.OAuth).filter_by( user_id=current_user.id, ) try: @@ -268,7 +268,7 @@ if ub.oauth_support: def unlink_oauth(provider): if request.host_url + 'me' != request.referrer: pass - query = ub.session.query(ub.OAuth).filter_by( + query = g.ubsession.query(ub.OAuth).filter_by( provider=provider, user_id=current_user.id, ) @@ -277,13 +277,13 @@ if ub.oauth_support: if current_user and current_user.is_authenticated: oauth_entry.user = current_user try: - ub.session.delete(oauth_entry) - ub.session.commit() + g.ubsession.delete(oauth_entry) + g.ubsession.commit() logout_oauth_user() flash(_(u"Unlink to %(oauth)s Succeeded", oauth=oauth_check[provider]), category="success") except Exception as e: log.exception(e) - ub.session.rollback() + g.ubsession.rollback() flash(_(u"Unlink to %(oauth)s Failed", oauth=oauth_check[provider]), category="error") except NoResultFound: log.warning("oauth %s for user %d not found", provider, current_user.id) diff --git a/cps/opds.py b/cps/opds.py index 05e9c68c..e92bd50a 100644 --- a/cps/opds.py +++ b/cps/opds.py @@ -33,7 +33,7 @@ from werkzeug.security import check_password_hash from . import constants, logger, config, db, calibre_db, ub, services, get_locale, isoLanguages from .helper import get_download_link, get_book_cover from .pagination import Pagination -from .web import render_read_books, download_required, load_user_from_request +from .web import render_read_books, load_user_from_request from flask_babel import gettext as _ from babel import Locale as LC from babel.core import UnknownLocaleError @@ -128,7 +128,7 @@ def feed_best_rated(): @requires_basic_auth_if_no_ano def feed_hot(): off = request.args.get("offset") or 0 - all_books = ub.session.query(ub.Downloads, func.count(ub.Downloads.book_id)).order_by( + all_books = g.ubsession.query(ub.Downloads, func.count(ub.Downloads.book_id)).order_by( func.count(ub.Downloads.book_id).desc()).group_by(ub.Downloads.book_id) hot_books = all_books.offset(off).limit(config.config_books_per_page) entries = list() @@ -361,17 +361,17 @@ def feed_shelfindex(): def feed_shelf(book_id): off = request.args.get("offset") or 0 if current_user.is_anonymous: - shelf = ub.session.query(ub.Shelf).filter(ub.Shelf.is_public == 1, + shelf = g.ubsession.query(ub.Shelf).filter(ub.Shelf.is_public == 1, ub.Shelf.id == book_id).first() else: - shelf = ub.session.query(ub.Shelf).filter(or_(and_(ub.Shelf.user_id == int(current_user.id), + shelf = g.ubsession.query(ub.Shelf).filter(or_(and_(ub.Shelf.user_id == int(current_user.id), ub.Shelf.id == book_id), and_(ub.Shelf.is_public == 1, ub.Shelf.id == book_id))).first() result = list() # user is allowed to access shelf if shelf: - books_in_shelf = ub.session.query(ub.BookShelf).filter(ub.BookShelf.shelf == book_id).order_by( + books_in_shelf = g.ubsession.query(ub.BookShelf).filter(ub.BookShelf.shelf == book_id).order_by( ub.BookShelf.order.asc()).all() for book in books_in_shelf: cur_book = calibre_db.get_book(book.book_id) @@ -427,7 +427,7 @@ def check_auth(username, password): username = username.encode('windows-1252') except UnicodeEncodeError: username = username.encode('utf-8') - user = ub.session.query(ub.User).filter(func.lower(ub.User.nickname) == + user = g.ubsession.query(ub.User).filter(func.lower(ub.User.nickname) == username.decode('utf-8').lower()).first() return bool(user and check_password_hash(str(user.password), password)) diff --git a/cps/shelf.py b/cps/shelf.py index ea7f1eeb..8ab0b5d8 100644 --- a/cps/shelf.py +++ b/cps/shelf.py @@ -23,7 +23,7 @@ from __future__ import division, print_function, unicode_literals from datetime import datetime -from flask import Blueprint, request, flash, redirect, url_for +from flask import Blueprint, request, flash, redirect, url_for, g from flask_babel import gettext as _ from flask_login import login_required, current_user from sqlalchemy.sql.expression import func @@ -60,7 +60,7 @@ def check_shelf_view_permissions(cur_shelf): @login_required def add_to_shelf(shelf_id, book_id): xhr = request.headers.get('X-Requested-With') == 'XMLHttpRequest' - shelf = ub.session.query(ub.Shelf).filter(ub.Shelf.id == shelf_id).first() + shelf = g.ubsession.query(ub.Shelf).filter(ub.Shelf.id == shelf_id).first() if shelf is None: log.error("Invalid shelf specified: %s", shelf_id) if not xhr: @@ -75,7 +75,7 @@ def add_to_shelf(shelf_id, book_id): return redirect(url_for('web.index')) return "Sorry you are not allowed to add a book to the the shelf: %s" % shelf.name, 403 - book_in_shelf = ub.session.query(ub.BookShelf).filter(ub.BookShelf.shelf == shelf_id, + book_in_shelf = g.ubsession.query(ub.BookShelf).filter(ub.BookShelf.shelf == shelf_id, ub.BookShelf.book_id == book_id).first() if book_in_shelf: log.error("Book %s is already part of %s", book_id, shelf) @@ -84,7 +84,7 @@ def add_to_shelf(shelf_id, book_id): return redirect(url_for('web.index')) return "Book is already part of the shelf: %s" % shelf.name, 400 - maxOrder = ub.session.query(func.max(ub.BookShelf.order)).filter(ub.BookShelf.shelf == shelf_id).first() + maxOrder = g.ubsession.query(func.max(ub.BookShelf.order)).filter(ub.BookShelf.shelf == shelf_id).first() if maxOrder[0] is None: maxOrder = 0 else: @@ -93,10 +93,10 @@ def add_to_shelf(shelf_id, book_id): shelf.books.append(ub.BookShelf(shelf=shelf.id, book_id=book_id, order=maxOrder + 1)) shelf.last_modified = datetime.utcnow() try: - ub.session.merge(shelf) - ub.session.commit() + g.ubsession.merge(shelf) + g.ubsession.commit() except (OperationalError, InvalidRequestError): - ub.session.rollback() + g.ubsession.rollback() flash(_(u"Settings DB is not Writeable"), category="error") if "HTTP_REFERER" in request.environ: return redirect(request.environ["HTTP_REFERER"]) @@ -114,7 +114,7 @@ def add_to_shelf(shelf_id, book_id): @shelf.route("/shelf/massadd/") @login_required def search_to_shelf(shelf_id): - shelf = ub.session.query(ub.Shelf).filter(ub.Shelf.id == shelf_id).first() + shelf = g.ubsession.query(ub.Shelf).filter(ub.Shelf.id == shelf_id).first() if shelf is None: log.error("Invalid shelf specified: %s", shelf_id) flash(_(u"Invalid shelf specified"), category="error") @@ -126,7 +126,7 @@ def search_to_shelf(shelf_id): if current_user.id in ub.searched_ids and ub.searched_ids[current_user.id]: books_for_shelf = list() - books_in_shelf = ub.session.query(ub.BookShelf).filter(ub.BookShelf.shelf == shelf_id).all() + books_in_shelf = g.ubsession.query(ub.BookShelf).filter(ub.BookShelf.shelf == shelf_id).all() if books_in_shelf: book_ids = list() for book_id in books_in_shelf: @@ -142,7 +142,7 @@ def search_to_shelf(shelf_id): flash(_(u"Books are already part of the shelf: %(name)s", name=shelf.name), category="error") return redirect(url_for('web.index')) - maxOrder = ub.session.query(func.max(ub.BookShelf.order)).filter(ub.BookShelf.shelf == shelf_id).first() + maxOrder = g.ubsession.query(func.max(ub.BookShelf.order)).filter(ub.BookShelf.shelf == shelf_id).first() if maxOrder[0] is None: maxOrder = 0 else: @@ -153,11 +153,11 @@ def search_to_shelf(shelf_id): shelf.books.append(ub.BookShelf(shelf=shelf.id, book_id=book, order=maxOrder)) shelf.last_modified = datetime.utcnow() try: - ub.session.merge(shelf) - ub.session.commit() + g.ubsession.merge(shelf) + g.ubsession.commit() flash(_(u"Books have been added to shelf: %(sname)s", sname=shelf.name), category="success") except (OperationalError, InvalidRequestError): - ub.session.rollback() + g.ubsession.rollback() flash(_(u"Settings DB is not Writeable"), category="error") else: flash(_(u"Could not add books to shelf: %(sname)s", sname=shelf.name), category="error") @@ -168,7 +168,7 @@ def search_to_shelf(shelf_id): @login_required def remove_from_shelf(shelf_id, book_id): xhr = request.headers.get('X-Requested-With') == 'XMLHttpRequest' - shelf = ub.session.query(ub.Shelf).filter(ub.Shelf.id == shelf_id).first() + shelf = g.ubsession.query(ub.Shelf).filter(ub.Shelf.id == shelf_id).first() if shelf is None: log.error("Invalid shelf specified: %s", shelf_id) if not xhr: @@ -184,7 +184,7 @@ def remove_from_shelf(shelf_id, book_id): # false 0 x 0 if check_shelf_edit_permissions(shelf): - book_shelf = ub.session.query(ub.BookShelf).filter(ub.BookShelf.shelf == shelf_id, + book_shelf = g.ubsession.query(ub.BookShelf).filter(ub.BookShelf.shelf == shelf_id, ub.BookShelf.book_id == book_id).first() if book_shelf is None: @@ -194,11 +194,11 @@ def remove_from_shelf(shelf_id, book_id): return "Book already removed from shelf", 410 try: - ub.session.delete(book_shelf) + g.ubsession.delete(book_shelf) shelf.last_modified = datetime.utcnow() - ub.session.commit() + g.ubsession.commit() except (OperationalError, InvalidRequestError): - ub.session.rollback() + g.ubsession.rollback() flash(_(u"Settings DB is not Writeable"), category="error") if "HTTP_REFERER" in request.environ: return redirect(request.environ["HTTP_REFERER"]) @@ -232,7 +232,7 @@ def create_shelf(): is_shelf_name_unique = False if shelf.is_public == 1: - is_shelf_name_unique = ub.session.query(ub.Shelf) \ + is_shelf_name_unique = g.ubsession.query(ub.Shelf) \ .filter((ub.Shelf.name == to_save["title"]) & (ub.Shelf.is_public == 1)) \ .first() is None @@ -240,7 +240,7 @@ def create_shelf(): flash(_(u"A public shelf with the name '%(title)s' already exists.", title=to_save["title"]), category="error") else: - is_shelf_name_unique = ub.session.query(ub.Shelf) \ + is_shelf_name_unique = g.ubsession.query(ub.Shelf) \ .filter((ub.Shelf.name == to_save["title"]) & (ub.Shelf.is_public == 0) & (ub.Shelf.user_id == int(current_user.id)))\ .first() is None @@ -251,15 +251,15 @@ def create_shelf(): if is_shelf_name_unique: try: - ub.session.add(shelf) - ub.session.commit() + g.ubsession.add(shelf) + g.ubsession.commit() flash(_(u"Shelf %(title)s created", title=to_save["title"]), category="success") return redirect(url_for('shelf.show_shelf', shelf_id=shelf.id)) except (OperationalError, InvalidRequestError): - ub.session.rollback() + g.ubsession.rollback() flash(_(u"Settings DB is not Writeable"), category="error") except Exception: - ub.session.rollback() + g.ubsession.rollback() flash(_(u"There was an error"), category="error") return render_title_template('shelf_edit.html', shelf=shelf, title=_(u"Create a Shelf"), page="shelfcreate") else: @@ -269,13 +269,13 @@ def create_shelf(): @shelf.route("/shelf/edit/", methods=["GET", "POST"]) @login_required def edit_shelf(shelf_id): - shelf = ub.session.query(ub.Shelf).filter(ub.Shelf.id == shelf_id).first() + shelf = g.ubsession.query(ub.Shelf).filter(ub.Shelf.id == shelf_id).first() if request.method == "POST": to_save = request.form.to_dict() is_shelf_name_unique = False if shelf.is_public == 1: - is_shelf_name_unique = ub.session.query(ub.Shelf) \ + is_shelf_name_unique = g.ubsession.query(ub.Shelf) \ .filter((ub.Shelf.name == to_save["title"]) & (ub.Shelf.is_public == 1)) \ .filter(ub.Shelf.id != shelf_id) \ .first() is None @@ -284,7 +284,7 @@ def edit_shelf(shelf_id): flash(_(u"A public shelf with the name '%(title)s' already exists.", title=to_save["title"]), category="error") else: - is_shelf_name_unique = ub.session.query(ub.Shelf) \ + is_shelf_name_unique = g.ubsession.query(ub.Shelf) \ .filter((ub.Shelf.name == to_save["title"]) & (ub.Shelf.is_public == 0) & (ub.Shelf.user_id == int(current_user.id)))\ .filter(ub.Shelf.id != shelf_id)\ @@ -302,13 +302,13 @@ def edit_shelf(shelf_id): else: shelf.is_public = 0 try: - ub.session.commit() + g.ubsession.commit() flash(_(u"Shelf %(title)s changed", title=to_save["title"]), category="success") except (OperationalError, InvalidRequestError): - ub.session.rollback() + g.ubsession.rollback() flash(_(u"Settings DB is not Writeable"), category="error") except Exception: - ub.session.rollback() + g.ubsession.rollback() flash(_(u"There was an error"), category="error") return render_title_template('shelf_edit.html', shelf=shelf, title=_(u"Edit a shelf"), page="shelfedit") else: @@ -319,10 +319,10 @@ def delete_shelf_helper(cur_shelf): if not cur_shelf or not check_shelf_edit_permissions(cur_shelf): return shelf_id = cur_shelf.id - ub.session.delete(cur_shelf) - ub.session.query(ub.BookShelf).filter(ub.BookShelf.shelf == shelf_id).delete() - ub.session.add(ub.ShelfArchive(uuid=cur_shelf.uuid, user_id=cur_shelf.user_id)) - ub.session.commit() + g.ubsession.delete(cur_shelf) + g.ubsession.query(ub.BookShelf).filter(ub.BookShelf.shelf == shelf_id).delete() + g.ubsession.add(ub.ShelfArchive(uuid=cur_shelf.uuid, user_id=cur_shelf.user_id)) + g.ubsession.commit() log.info("successfully deleted %s", cur_shelf) @@ -330,11 +330,11 @@ def delete_shelf_helper(cur_shelf): @shelf.route("/shelf/delete/") @login_required def delete_shelf(shelf_id): - cur_shelf = ub.session.query(ub.Shelf).filter(ub.Shelf.id == shelf_id).first() + cur_shelf = g.ubsession.query(ub.Shelf).filter(ub.Shelf.id == shelf_id).first() try: delete_shelf_helper(cur_shelf) except (OperationalError, InvalidRequestError): - ub.session.rollback() + g.ubsession.rollback() flash(_(u"Settings DB is not Writeable"), category="error") return redirect(url_for('web.index')) @@ -343,14 +343,14 @@ def delete_shelf(shelf_id): @shelf.route("/shelf//") @login_required_if_no_ano def show_shelf(shelf_type, shelf_id): - shelf = ub.session.query(ub.Shelf).filter(ub.Shelf.id == shelf_id).first() + shelf = g.ubsession.query(ub.Shelf).filter(ub.Shelf.id == shelf_id).first() result = list() # user is allowed to access shelf if shelf and check_shelf_view_permissions(shelf): page = "shelf.html" if shelf_type == 1 else 'shelfdown.html' - books_in_shelf = ub.session.query(ub.BookShelf).filter(ub.BookShelf.shelf == shelf_id)\ + books_in_shelf = g.ubsession.query(ub.BookShelf).filter(ub.BookShelf.shelf == shelf_id)\ .order_by(ub.BookShelf.order.asc()).all() for book in books_in_shelf: cur_book = calibre_db.get_filtered_book(book.book_id) @@ -361,10 +361,10 @@ def show_shelf(shelf_type, shelf_id): if not cur_book: log.info('Not existing book %s in %s deleted', book.book_id, shelf) try: - ub.session.query(ub.BookShelf).filter(ub.BookShelf.book_id == book.book_id).delete() - ub.session.commit() + g.ubsession.query(ub.BookShelf).filter(ub.BookShelf.book_id == book.book_id).delete() + g.ubsession.commit() except (OperationalError, InvalidRequestError): - ub.session.rollback() + g.ubsession.rollback() flash(_(u"Settings DB is not Writeable"), category="error") return render_title_template(page, entries=result, title=_(u"Shelf: '%(name)s'", name=shelf.name), shelf=shelf, page="shelf") @@ -378,7 +378,7 @@ def show_shelf(shelf_type, shelf_id): def order_shelf(shelf_id): if request.method == "POST": to_save = request.form.to_dict() - books_in_shelf = ub.session.query(ub.BookShelf).filter(ub.BookShelf.shelf == shelf_id).order_by( + books_in_shelf = g.ubsession.query(ub.BookShelf).filter(ub.BookShelf.shelf == shelf_id).order_by( ub.BookShelf.order.asc()).all() counter = 0 for book in books_in_shelf: @@ -386,15 +386,15 @@ def order_shelf(shelf_id): counter += 1 # if order diffrent from before -> shelf.last_modified = datetime.utcnow() try: - ub.session.commit() + g.ubsession.commit() except (OperationalError, InvalidRequestError): - ub.session.rollback() + g.ubsession.rollback() flash(_(u"Settings DB is not Writeable"), category="error") - shelf = ub.session.query(ub.Shelf).filter(ub.Shelf.id == shelf_id).first() + shelf = g.ubsession.query(ub.Shelf).filter(ub.Shelf.id == shelf_id).first() result = list() if shelf and check_shelf_view_permissions(shelf): - books_in_shelf2 = ub.session.query(ub.BookShelf).filter(ub.BookShelf.shelf == shelf_id) \ + books_in_shelf2 = g.ubsession.query(ub.BookShelf).filter(ub.BookShelf.shelf == shelf_id) \ .order_by(ub.BookShelf.order.asc()).all() for book in books_in_shelf2: cur_book = calibre_db.get_filtered_book(book.book_id) diff --git a/cps/ub.py b/cps/ub.py index 145575d1..9a7069dc 100644 --- a/cps/ub.py +++ b/cps/ub.py @@ -46,12 +46,13 @@ from sqlalchemy import String, Integer, SmallInteger, Boolean, DateTime, Float, from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.orm.attributes import flag_modified from sqlalchemy.orm import backref, relationship, sessionmaker, Session +from sqlalchemy.orm import relationship, scoped_session from werkzeug.security import generate_password_hash from . import constants -session = None +Scoped_Session = None app_DB_path = None Base = declarative_base() searched_ids = {} @@ -219,9 +220,9 @@ class UserBase: except AttributeError: pass try: - session.commit() + g.ubsession.commit() except (exc.OperationalError, exc.InvalidRequestError): - session.rollback() + g.ubsession.rollback() # ToDo: Error message def __repr__(self): @@ -279,6 +280,7 @@ class Anonymous(AnonymousUserMixin, UserBase): self.loadSettings() def loadSettings(self): + session = Scoped_Session() data = session.query(User).filter(User.role.op('&')(constants.ROLE_ANONYMOUS) == constants.ROLE_ANONYMOUS)\ .first() # type: User self.nickname = data.nickname @@ -297,6 +299,7 @@ class Anonymous(AnonymousUserMixin, UserBase): # Initialize flask_session once if 'view' not in flask_session: flask_session['view']={} + session.close() def role_admin(self): @@ -673,18 +676,18 @@ def clean_database(session): # Save downloaded books per user in calibre-web's own database def update_download(book_id, user_id): - check = session.query(Downloads).filter(Downloads.user_id == user_id).filter(Downloads.book_id == book_id).first() + check = g.ubsession.query(Downloads).filter(Downloads.user_id == user_id).filter(Downloads.book_id == book_id).first() if not check: new_download = Downloads(user_id=user_id, book_id=book_id) - session.add(new_download) - session.commit() + g.ubsession.add(new_download) + g.ubsession.commit() # Delete non exisiting downloaded books in calibre-web's own database def delete_download(book_id): - session.query(Downloads).filter(book_id == Downloads.book_id).delete() - session.commit() + g.ubsession.query(Downloads).filter(book_id == Downloads.book_id).delete() + g.ubsession.commit() # Generate user Guest (translated text), as anonymous user, no rights def create_anonymous_user(session): @@ -716,18 +719,21 @@ def create_admin_user(session): except Exception: session.rollback() +def create_session(): + pass def init_db(app_db_path): # Open session for database connection - global session + global Scoped_Session global app_DB_path + global engine app_DB_path = app_db_path engine = create_engine(u'sqlite:///{0}'.format(app_db_path), echo=False) - Session = sessionmaker() - Session.configure(bind=engine) - session = Session() + Scoped_Session = scoped_session(sessionmaker()) # sessionmaker() + Scoped_Session.configure(bind=engine) + session = Scoped_Session() if os.path.exists(app_db_path): Base.metadata.create_all(engine) @@ -737,6 +743,7 @@ def init_db(app_db_path): Base.metadata.create_all(engine) create_admin_user(session) create_anonymous_user(session) + session.close() def dispose(): diff --git a/cps/web.py b/cps/web.py index f41c3e0d..15d22d60 100644 --- a/cps/web.py +++ b/cps/web.py @@ -139,6 +139,7 @@ def add_security_headers(resp): resp.headers['X-XSS-Protection'] = '1; mode=block' resp.headers['Strict-Transport-Security'] = 'max-age=31536000; includeSubDomains' # log.debug(request.full_path) + g.ubsession.close() return resp web = Blueprint('web', __name__) @@ -147,12 +148,13 @@ log = logger.create() # ################################### Login logic and rights management ############################################### def _fetch_user_by_name(username): - return ub.session.query(ub.User).filter(func.lower(ub.User.nickname) == username.lower()).first() + return g.ubsession.query(ub.User).filter(func.lower(ub.User.nickname) == username.lower()).first() @lm.user_loader def load_user(user_id): - return ub.session.query(ub.User).filter(ub.User.id == int(user_id)).first() + g.ubsession = ub.Scoped_Session() + return g.ubsession.query(ub.User).filter(ub.User.id == int(user_id)).first() @lm.request_loader @@ -291,6 +293,7 @@ def edit_required(f): @web.before_app_request def before_request(): + g.ubsession = ub.Scoped_Session() if current_user.is_authenticated: confirm_login() g.constants = constants @@ -300,7 +303,7 @@ def before_request(): g.allow_upload = config.config_uploading g.current_theme = config.config_theme g.config_authors_max = config.config_authors_max - g.shelves_access = ub.session.query(ub.Shelf).filter( + g.shelves_access = g.ubsession.query(ub.Shelf).filter( or_(ub.Shelf.is_public == 1, ub.Shelf.user_id == current_user.id)).order_by(ub.Shelf.name).all() if not config.db_configured and request.endpoint not in ( 'admin.basic_configuration', 'login') and '/static/' not in request.path: @@ -350,8 +353,7 @@ def import_ldap_users(): username = user_data[user_login_field][0].decode('utf-8') # check for duplicate username - if ub.session.query(ub.User).filter(func.lower(ub.User.nickname) == username.lower()).first(): - # if ub.session.query(ub.User).filter(ub.User.nickname == username).first(): + if g.ubsession.query(ub.User).filter(func.lower(ub.User.nickname) == username.lower()).first(): log.warning("LDAP User %s Already in Database", user_data) continue @@ -365,7 +367,7 @@ def import_ldap_users(): log.debug('No Mail Field Found in LDAP Response') useremail = username + '@email.com' # check for duplicate email - if ub.session.query(ub.User).filter(func.lower(ub.User.email) == useremail.lower()).first(): + if g.ubsession.query(ub.User).filter(func.lower(ub.User.email) == useremail.lower()).first(): log.warning("LDAP Email %s Already in Database", user_data) continue content = ub.User() @@ -379,13 +381,13 @@ def import_ldap_users(): content.denied_tags = config.config_denied_tags content.allowed_column_value = config.config_allowed_column_value content.denied_column_value = config.config_denied_column_value - ub.session.add(content) + g.ubsession.add(content) try: - ub.session.commit() + g.ubsession.commit() imported +=1 except Exception as e: log.warning("Failed to create LDAP user: %s - %s", user, e) - ub.session.rollback() + g.ubsession.rollback() showtext['text'] = _(u'Failed to Create at Least One LDAP User') else: log.warning("LDAP User: %s Not Found", user) @@ -428,19 +430,19 @@ def get_email_status_json(): @login_required def bookmark(book_id, book_format): bookmark_key = request.form["bookmark"] - ub.session.query(ub.Bookmark).filter(and_(ub.Bookmark.user_id == int(current_user.id), + g.ubsession.query(ub.Bookmark).filter(and_(ub.Bookmark.user_id == int(current_user.id), ub.Bookmark.book_id == book_id, ub.Bookmark.format == book_format)).delete() if not bookmark_key: - ub.session.commit() + g.ubsession.commit() return "", 204 lbookmark = ub.Bookmark(user_id=current_user.id, book_id=book_id, format=book_format, bookmark_key=bookmark_key) - ub.session.merge(lbookmark) - ub.session.commit() + g.ubsession.merge(lbookmark) + g.ubsession.commit() return "", 201 @@ -448,7 +450,7 @@ def bookmark(book_id, book_format): @login_required def toggle_read(book_id): if not config.config_read_column: - book = ub.session.query(ub.ReadBook).filter(and_(ub.ReadBook.user_id == int(current_user.id), + book = g.ubsession.query(ub.ReadBook).filter(and_(ub.ReadBook.user_id == int(current_user.id), ub.ReadBook.book_id == book_id)).first() if book: if book.read_status == ub.ReadBook.STATUS_FINISHED: @@ -464,8 +466,8 @@ def toggle_read(book_id): kobo_reading_state.current_bookmark = ub.KoboBookmark() kobo_reading_state.statistics = ub.KoboStatistics() book.kobo_reading_state = kobo_reading_state - ub.session.merge(book) - ub.session.commit() + g.ubsession.merge(book) + g.ubsession.commit() else: try: calibre_db.update_title_sort(config) @@ -490,7 +492,7 @@ def toggle_read(book_id): @web.route("/ajax/togglearchived/", methods=['POST']) @login_required def toggle_archived(book_id): - archived_book = ub.session.query(ub.ArchivedBook).filter(and_(ub.ArchivedBook.user_id == int(current_user.id), + archived_book = g.ubsession.query(ub.ArchivedBook).filter(and_(ub.ArchivedBook.user_id == int(current_user.id), ub.ArchivedBook.book_id == book_id)).first() if archived_book: archived_book.is_archived = not archived_book.is_archived @@ -498,8 +500,8 @@ def toggle_archived(book_id): else: archived_book = ub.ArchivedBook(user_id=current_user.id, book_id=book_id) archived_book.is_archived = True - ub.session.merge(archived_book) - ub.session.commit() + g.ubsession.merge(archived_book) + g.ubsession.commit() return "" @@ -738,7 +740,7 @@ def render_hot_books(page): else: random = false() off = int(int(config.config_books_per_page) * (page - 1)) - all_books = ub.session.query(ub.Downloads, func.count(ub.Downloads.book_id)).order_by( + all_books = g.ubsession.query(ub.Downloads, func.count(ub.Downloads.book_id)).order_by( func.count(ub.Downloads.book_id).desc()).group_by(ub.Downloads.book_id) hot_books = all_books.offset(off).limit(config.config_books_per_page) entries = list() @@ -749,8 +751,6 @@ def render_hot_books(page): entries.append(downloadBook) else: ub.delete_download(book.Downloads.book_id) - # ub.session.query(ub.Downloads).filter(book.Downloads.book_id == ub.Downloads.book_id).delete() - # ub.session.commit() numBooks = entries.__len__() pagination = Pagination(page, config.config_books_per_page, numBooks) return render_title_template('index.html', random=random, entries=entries, pagination=pagination, @@ -953,7 +953,7 @@ def render_read_books(page, are_read, as_xml=False, order=None, *args, **kwargs) def render_archived_books(page, order): order = order or [] archived_books = ( - ub.session.query(ub.ArchivedBook) + g.ubsession.query(ub.ArchivedBook) .filter(ub.ArchivedBook.user_id == int(current_user.id)) .filter(ub.ArchivedBook.is_archived == True) .all() @@ -1085,7 +1085,7 @@ def update_table_settings(): flag_modified(current_user, "view_settings") except AttributeError: pass - ub.session.commit() + g.ubsession.commit() except InvalidRequestError: log.error("Invalid request received: %r ", request, ) return "Invalid request", 400 @@ -1550,9 +1550,9 @@ def register(): return render_title_template('register.html', title=_(u"register"), page="register") - existing_user = ub.session.query(ub.User).filter(func.lower(ub.User.nickname) == nickname + existing_user = g.ubsession.query(ub.User).filter(func.lower(ub.User.nickname) == nickname .lower()).first() - existing_email = ub.session.query(ub.User).filter(ub.User.email == to_save["email"].lower()).first() + existing_email = g.ubsession.query(ub.User).filter(ub.User.email == to_save["email"].lower()).first() if not existing_user and not existing_email: content = ub.User() if check_valid_domain(to_save["email"]): @@ -1563,13 +1563,13 @@ def register(): content.role = config.config_default_role content.sidebar_view = config.config_default_show try: - ub.session.add(content) - ub.session.commit() + g.ubsession.add(content) + g.ubsession.commit() if feature_support['oauth']: register_user_with_oauth(content) send_registration_mail(to_save["email"], nickname, password) except Exception: - ub.session.rollback() + g.ubsession.rollback() flash(_(u"An unknown error occurred. Please try again later."), category="error") return render_title_template('register.html', title=_(u"register"), page="register") else: @@ -1599,7 +1599,7 @@ def login(): flash(_(u"Cannot activate LDAP authentication"), category="error") if request.method == "POST": form = request.form.to_dict() - user = ub.session.query(ub.User).filter(func.lower(ub.User.nickname) == form['username'].strip().lower()) \ + user = g.ubsession.query(ub.User).filter(func.lower(ub.User.nickname) == form['username'].strip().lower()) \ .first() if config.config_login_type == constants.LOGIN_LDAP and services.ldap and user and form['password'] != "": login_result, error = services.ldap.bind_user(form['username'], form['password']) @@ -1675,8 +1675,8 @@ def logout(): @remote_login_required def remote_login(): auth_token = ub.RemoteAuthToken() - ub.session.add(auth_token) - ub.session.commit() + g.ubsession.add(auth_token) + g.ubsession.commit() verify_url = url_for('web.verify_token', token=auth_token.auth_token, _external=true) log.debug(u"Remot Login request with token: %s", auth_token.auth_token) @@ -1688,7 +1688,7 @@ def remote_login(): @remote_login_required @login_required def verify_token(token): - auth_token = ub.session.query(ub.RemoteAuthToken).filter(ub.RemoteAuthToken.auth_token == token).first() + auth_token = g.ubsession.query(ub.RemoteAuthToken).filter(ub.RemoteAuthToken.auth_token == token).first() # Token not found if auth_token is None: @@ -1698,8 +1698,8 @@ def verify_token(token): # Token expired if datetime.now() > auth_token.expiration: - ub.session.delete(auth_token) - ub.session.commit() + g.ubsession.delete(auth_token) + g.ubsession.commit() flash(_(u"Token has expired"), category="error") log.error(u"Remote Login token expired") @@ -1708,7 +1708,7 @@ def verify_token(token): # Update token with user information auth_token.user_id = current_user.id auth_token.verified = True - ub.session.commit() + g.ubsession.commit() flash(_(u"Success! Please return to your device"), category="success") log.debug(u"Remote Login token for userid %s verified", auth_token.user_id) @@ -1719,7 +1719,7 @@ def verify_token(token): @remote_login_required def token_verified(): token = request.form['token'] - auth_token = ub.session.query(ub.RemoteAuthToken).filter(ub.RemoteAuthToken.auth_token == token).first() + auth_token = g.ubsession.query(ub.RemoteAuthToken).filter(ub.RemoteAuthToken.auth_token == token).first() data = {} @@ -1730,8 +1730,8 @@ def token_verified(): # Token expired elif datetime.now() > auth_token.expiration: - ub.session.delete(auth_token) - ub.session.commit() + g.ubsession.delete(auth_token) + g.ubsession.commit() data['status'] = 'error' data['message'] = _(u"Token has expired") @@ -1740,11 +1740,11 @@ def token_verified(): data['status'] = 'not_verified' else: - user = ub.session.query(ub.User).filter(ub.User.id == auth_token.user_id).first() + user = g.ubsession.query(ub.User).filter(ub.User.id == auth_token.user_id).first() login_user(user) - ub.session.delete(auth_token) - ub.session.commit() + g.ubsession.delete(auth_token) + g.ubsession.commit() data['status'] = 'success' log.debug(u"Remote Login for userid %s succeded", user.id) @@ -1800,7 +1800,7 @@ def profile(): current_user.email = to_save["email"] if "nickname" in to_save and to_save["nickname"] != current_user.nickname: # Query User nickname, if not existing, change - if not ub.session.query(ub.User).filter(ub.User.nickname == to_save["nickname"]).scalar(): + if not g.ubsession.query(ub.User).filter(ub.User.nickname == to_save["nickname"]).scalar(): current_user.nickname = to_save["nickname"] else: flash(_(u"This username is already taken"), category="error") @@ -1829,11 +1829,11 @@ def profile(): current_user.sidebar_view += constants.DETAIL_RANDOM try: - ub.session.commit() + g.ubsession.commit() flash(_(u"Profile updated"), category="success") log.debug(u"Profile updated") except IntegrityError: - ub.session.rollback() + g.ubsession.rollback() flash(_(u"Found an existing account for this e-mail address."), category="error") log.debug(u"Found an existing account for this e-mail address.") '''return render_title_template("user_edit.html", @@ -1872,7 +1872,7 @@ def read_book(book_id, book_format): # check if book has bookmark bookmark = None if current_user.is_authenticated: - bookmark = ub.session.query(ub.Bookmark).filter(and_(ub.Bookmark.user_id == int(current_user.id), + bookmark = g.ubsession.query(ub.Bookmark).filter(and_(ub.Bookmark.user_id == int(current_user.id), ub.Bookmark.book_id == book_id, ub.Bookmark.format == book_format.upper())).first() if book_format.lower() == "epub": @@ -1924,13 +1924,13 @@ def show_book(book_id): isoLanguages.get(part3=entries.languages[index].lang_code).name) cc = get_cc_columns(filter_config_custom_read=True) book_in_shelfs = [] - shelfs = ub.session.query(ub.BookShelf).filter(ub.BookShelf.book_id == book_id).all() + shelfs = g.ubsession.query(ub.BookShelf).filter(ub.BookShelf.book_id == book_id).all() for entry in shelfs: book_in_shelfs.append(entry.shelf) if not current_user.is_anonymous: if not config.config_read_column: - matching_have_read_book = ub.session.query(ub.ReadBook). \ + matching_have_read_book = g.ubsession.query(ub.ReadBook). \ filter(and_(ub.ReadBook.user_id == int(current_user.id), ub.ReadBook.book_id == book_id)).all() have_read = len( matching_have_read_book) > 0 and matching_have_read_book[0].read_status == ub.ReadBook.STATUS_FINISHED @@ -1942,7 +1942,7 @@ def show_book(book_id): log.error("Custom Column No.%d is not existing in calibre database", config.config_read_column) have_read = None - archived_book = ub.session.query(ub.ArchivedBook).\ + archived_book = g.ubsession.query(ub.ArchivedBook).\ filter(and_(ub.ArchivedBook.user_id == int(current_user.id), ub.ArchivedBook.book_id == book_id)).first() is_archived = archived_book and archived_book.is_archived From f13522559d1b7749037e2cecb2033a1425218716 Mon Sep 17 00:00:00 2001 From: Ozzieisaacs Date: Mon, 7 Dec 2020 13:06:29 +0100 Subject: [PATCH 2/7] Fixed problems on startup with config session --- cps/__init__.py | 5 ++--- cps/config_sql.py | 28 +++++++++++++++------------- 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/cps/__init__.py b/cps/__init__.py index 575a5a8a..1f28b918 100644 --- a/cps/__init__.py +++ b/cps/__init__.py @@ -71,7 +71,7 @@ lm.session_protection = 'strong' ub.init_db(cli.settingspath) # pylint: disable=no-member -config = config_sql.load_configuration(ub.Scoped_Session) +config = config_sql.load_configuration() web_server = WebServer() @@ -100,7 +100,7 @@ def create_app(): log.info('Starting Calibre Web...') Principal(app) lm.init_app(app) - app.secret_key = os.getenv('SECRET_KEY', config_sql.get_flask_session_key(ub.Scoped_Session)) + app.secret_key = os.getenv('SECRET_KEY', config.get_flask_session_key()) web_server.init_app(app, config) @@ -114,7 +114,6 @@ def create_app(): services.goodreads_support.connect(config.config_goodreads_api_key, config.config_goodreads_api_secret, config.config_use_goodreads) - return app @babel.localeselector diff --git a/cps/config_sql.py b/cps/config_sql.py index eabe4ba7..3e531eef 100644 --- a/cps/config_sql.py +++ b/cps/config_sql.py @@ -139,6 +139,7 @@ class _ConfigSQL(object): # pylint: disable=no-member def __init__(self, session): self._session = session + self._session.expire_on_commit = False self._settings = None self.db_configured = None self.config_calibre_dir = None @@ -241,6 +242,14 @@ class _ConfigSQL(object): def get_mail_server_configured(self): return not bool(self.mail_server == constants.DEFAULT_MAIL_SERVER) + def get_flask_session_key(self): + flask_settings = self._session.query(_Flask_Settings).one_or_none() + if flask_settings == None: + flask_settings = _Flask_Settings(os.urandom(32)) + self._session.add(flask_settings) + self._session.commit() + return flask_settings.flask_session_key + def set_from_dictionary(self, dictionary, field, convertor=None, default=None, encode=None): '''Possibly updates a field of this object. @@ -273,6 +282,10 @@ class _ConfigSQL(object): def load(self): '''Load all configuration values from the underlying storage.''' + #own = False + #if not self._session: + # self._session = ub.Scoped_Session() + # own = True s = self._read_from_storage() # type: _Settings for k, v in s.__dict__.items(): if k[0] != '_': @@ -395,8 +408,8 @@ def _migrate_database(session): _migrate_table(session, _Flask_Settings) -def load_configuration(Session): - session = Session() +def load_configuration(): + session = ub.Scoped_Session() _migrate_database(session) if not session.query(_Settings).count(): @@ -410,15 +423,4 @@ def load_configuration(Session): session.query(ub.User).filter(ub.User.mature_content != True). \ update({"denied_tags": conf.config_mature_content_tags}, synchronize_session=False) session.commit() - session.close() return conf - -def get_flask_session_key(Session): - session = Session() - flask_settings = session.query(_Flask_Settings).one_or_none() - if flask_settings == None: - flask_settings = _Flask_Settings(os.urandom(32)) - session.add(flask_settings) - session.commit() - session.close() - return flask_settings.flask_session_key From 5e3618716d2dd7ea71ecd041c5685e0fdb4bbc27 Mon Sep 17 00:00:00 2001 From: Ozzieisaacs Date: Mon, 7 Dec 2020 19:53:34 +0100 Subject: [PATCH 3/7] Fix missing session rollback on commit error --- cps/admin.py | 95 +++++++++++++++++++++++++++++++++++++---------- cps/config_sql.py | 34 +++++++++++++---- cps/kobo.py | 42 ++++++++++++++++----- cps/kobo_auth.py | 11 +++++- cps/oauth_bb.py | 11 +++++- cps/shelf.py | 7 +++- cps/ub.py | 14 +++++-- cps/web.py | 56 +++++++++++++++++++--------- 8 files changed, 208 insertions(+), 62 deletions(-) diff --git a/cps/admin.py b/cps/admin.py index f0a82e77..9b90a74e 100644 --- a/cps/admin.py +++ b/cps/admin.py @@ -206,7 +206,10 @@ def edit_domain(allow): vals = request.form.to_dict() answer = g.ubsession.query(ub.Registration).filter(ub.Registration.id == vals['pk']).first() answer.domain = vals['value'].replace('*', '%').replace('?', '_').lower() - g.ubsession.commit() + try: + g.ubsession.commit() + except OperationalError: + g.ubsession.rollback() return "" @@ -220,7 +223,10 @@ def add_domain(allow): if not check: new_domain = ub.Registration(domain=domain_name, allow=allow) g.ubsession.add(new_domain) - g.ubsession.commit() + try: + g.ubsession.commit() + except OperationalError: + g.ubsession.rollback() return "" @@ -230,12 +236,18 @@ def add_domain(allow): def delete_domain(): domain_id = request.form.to_dict()['domainid'].replace('*', '%').replace('?', '_').lower() g.ubsession.query(ub.Registration).filter(ub.Registration.id == domain_id).delete() - g.ubsession.commit() + try: + g.ubsession.commit() + except OperationalError: + g.ubsession.rollback() # If last domain was deleted, add all domains by default if not g.ubsession.query(ub.Registration).filter(ub.Registration.allow==1).count(): new_domain = ub.Registration(domain="%.%",allow=1) g.ubsession.add(new_domain) - g.ubsession.commit() + try: + g.ubsession.commit() + except OperationalError: + g.ubsession.rollback() return "" @@ -275,7 +287,10 @@ def edit_restriction(res_type): elementlist = usr.list_allowed_tags() elementlist[int(element['id'][1:])]=element['Element'] usr.allowed_tags = ','.join(elementlist) - g.ubsession.commit() + try: + g.ubsession.commit() + except OperationalError: + g.ubsession.rollback() if res_type == 3: # CColumn per user usr_id = os.path.split(request.referrer)[-1] if usr_id.isdigit() == True: @@ -285,7 +300,10 @@ def edit_restriction(res_type): elementlist = usr.list_allowed_column_values() elementlist[int(element['id'][1:])]=element['Element'] usr.allowed_column_value = ','.join(elementlist) - g.ubsession.commit() + try: + g.ubsession.commit() + except OperationalError: + g.ubsession.rollback() if element['id'].startswith('d'): if res_type == 0: # Tags as template elementlist = config.list_denied_tags() @@ -306,7 +324,10 @@ def edit_restriction(res_type): elementlist = usr.list_denied_tags() elementlist[int(element['id'][1:])]=element['Element'] usr.denied_tags = ','.join(elementlist) - g.ubsession.commit() + try: + g.ubsession.commit() + except OperationalError: + g.ubsession.rollback() if res_type == 3: # CColumn per user usr_id = os.path.split(request.referrer)[-1] if usr_id.isdigit() == True: @@ -316,7 +337,10 @@ def edit_restriction(res_type): elementlist = usr.list_denied_column_values() elementlist[int(element['id'][1:])]=element['Element'] usr.denied_column_value = ','.join(elementlist) - g.ubsession.commit() + try: + g.ubsession.commit() + except OperationalError: + g.ubsession.rollback() return "" def restriction_addition(element, list_func): @@ -362,10 +386,16 @@ def add_restriction(res_type): usr = current_user if 'submit_allow' in element: usr.allowed_tags = restriction_addition(element, usr.list_allowed_tags) - g.ubsession.commit() + try: + g.ubsession.commit() + except OperationalError: + g.ubsession.rollback() elif 'submit_deny' in element: usr.denied_tags = restriction_addition(element, usr.list_denied_tags) - g.ubsession.commit() + try: + g.ubsession.commit() + except OperationalError: + g.ubsession.rollback() if res_type == 3: # CustomC per user usr_id = os.path.split(request.referrer)[-1] if usr_id.isdigit() == True: @@ -374,10 +404,16 @@ def add_restriction(res_type): usr = current_user if 'submit_allow' in element: usr.allowed_column_value = restriction_addition(element, usr.list_allowed_column_values) - g.ubsession.commit() + try: + g.ubsession.commit() + except OperationalError: + g.ubsession.rollback() elif 'submit_deny' in element: usr.denied_column_value = restriction_addition(element, usr.list_denied_column_values) - g.ubsession.commit() + try: + g.ubsession.commit() + except OperationalError: + g.ubsession.rollback() return "" @admi.route("/ajax/deleterestriction/", methods=['POST']) @@ -407,10 +443,16 @@ def delete_restriction(res_type): usr = current_user if element['id'].startswith('a'): usr.allowed_tags = restriction_deletion(element, usr.list_allowed_tags) - g.ubsession.commit() + try: + g.ubsession.commit() + except OperationalError: + g.ubsession.rollback() elif element['id'].startswith('d'): usr.denied_tags = restriction_deletion(element, usr.list_denied_tags) - g.ubsession.commit() + try: + g.ubsession.commit() + except OperationalError: + g.ubsession.rollback() elif res_type == 3: # Columns per user usr_id = os.path.split(request.referrer)[-1] if usr_id.isdigit() == True: # select current user if admins are editing their own rights @@ -419,10 +461,16 @@ def delete_restriction(res_type): usr = current_user if element['id'].startswith('a'): usr.allowed_column_value = restriction_deletion(element, usr.list_allowed_column_values) - g.ubsession.commit() + try: + g.ubsession.commit() + except OperationalError: + g.ubsession.rollback() elif element['id'].startswith('d'): usr.denied_column_value = restriction_deletion(element, usr.list_denied_column_values) - g.ubsession.commit() + try: + g.ubsession.commit() + except OperationalError: + g.ubsession.rollback() return "" @@ -815,7 +863,10 @@ def _handle_new_user(to_save, content,languages, translations, kobo_support): content.allowed_column_value = config.config_allowed_column_value content.denied_column_value = config.config_denied_column_value g.ubsession.add(content) - g.ubsession.commit() + try: + g.ubsession.commit() + except OperationalError: + g.ubsession.rollback() flash(_(u"User '%(user)s' created", user=content.nickname), category="success") return redirect(url_for('admin.admin')) except IntegrityError: @@ -831,7 +882,10 @@ def _handle_edit_user(to_save, content,languages, translations, kobo_support): if g.ubsession.query(ub.User).filter(ub.User.role.op('&')(constants.ROLE_ADMIN) == constants.ROLE_ADMIN, ub.User.id != content.id).count(): g.ubsession.query(ub.User).filter(ub.User.id == content.id).delete() - g.ubsession.commit() + try: + g.ubsession.commit() + except OperationalError: + g.ubsession.rollback() flash(_(u"User '%(nick)s' deleted", nick=content.nickname), category="success") return redirect(url_for('admin.admin')) else: @@ -906,7 +960,10 @@ def _handle_edit_user(to_save, content,languages, translations, kobo_support): if "kindle_mail" in to_save and to_save["kindle_mail"] != content.kindle_mail: content.kindle_mail = to_save["kindle_mail"] try: - g.ubsession.commit() + try: + g.ubsession.commit() + except OperationalError: + g.ubsession.rollback() flash(_(u"User '%(nick)s' updated", nick=content.nickname), category="success") except IntegrityError: g.ubsession.rollback() diff --git a/cps/config_sql.py b/cps/config_sql.py index 3e531eef..bfcb81ad 100644 --- a/cps/config_sql.py +++ b/cps/config_sql.py @@ -22,8 +22,8 @@ import os import sys from sqlalchemy import exc, Column, String, Integer, SmallInteger, Boolean, BLOB, JSON +from sqlalchemy.exc import OperationalError from sqlalchemy.ext.declarative import declarative_base - from . import constants, cli, logger, ub @@ -247,7 +247,10 @@ class _ConfigSQL(object): if flask_settings == None: flask_settings = _Flask_Settings(os.urandom(32)) self._session.add(flask_settings) - self._session.commit() + try: + self._session.commit() + except OperationalError: + self._session.rollback() return flask_settings.flask_session_key @@ -308,7 +311,11 @@ class _ConfigSQL(object): log.warning("Log path %s not valid, falling back to default", self.config_logfile) self.config_logfile = logfile self._session.merge(s) - self._session.commit() + try: + self._session.commit() + except OperationalError as e: + log.error('Database error: %s', e) + self._session.rollback() def save(self): '''Apply all configuration values to the underlying storage.''' @@ -322,7 +329,11 @@ class _ConfigSQL(object): log.debug("_ConfigSQL updating storage") self._session.merge(s) - self._session.commit() + try: + self._session.commit() + except OperationalError as e: + log.error('Database error: %s', e) + self._session.rollback() self.load() def invalidate(self, error=None): @@ -363,7 +374,10 @@ def _migrate_table(session, orm_class): changed = True if changed: - session.commit() + try: + session.commit() + except OperationalError: + session.rollback() def autodetect_calibre_binary(): @@ -414,7 +428,10 @@ def load_configuration(): if not session.query(_Settings).count(): session.add(_Settings()) - session.commit() + try: + session.commit() + except OperationalError: + session.rollback() conf = _ConfigSQL(session) # Migrate from global restrictions to user based restrictions if bool(conf.config_default_show & constants.MATURE_CONTENT) and conf.config_denied_tags == "": @@ -422,5 +439,8 @@ def load_configuration(): conf.save() session.query(ub.User).filter(ub.User.mature_content != True). \ update({"denied_tags": conf.config_mature_content_tags}, synchronize_session=False) - session.commit() + try: + session.commit() + except OperationalError: + session.rollback() return conf diff --git a/cps/kobo.py b/cps/kobo.py index 47d10417..6c994525 100644 --- a/cps/kobo.py +++ b/cps/kobo.py @@ -44,6 +44,7 @@ from flask_login import current_user from werkzeug.datastructures import Headers from sqlalchemy import func from sqlalchemy.sql.expression import and_, or_ +from sqlalchemy.exc import OperationalError from sqlalchemy.orm import load_only from sqlalchemy.exc import StatementError import requests @@ -452,8 +453,10 @@ def HandleTagCreate(): items_unknown_to_calibre = add_items_to_shelf(items, shelf) if items_unknown_to_calibre: log.debug("Received request to add unknown books to a collection. Silently ignoring items.") - g.ubsession.commit() - + try: + g.ubsession.commit() + except OperationalError: + g.ubsession.rollback() return make_response(jsonify(str(shelf.uuid)), 201) @@ -485,7 +488,10 @@ def HandleTagUpdate(tag_id): shelf.name = name g.ubsession.merge(shelf) - g.ubsession.commit() + try: + g.ubsession.commit() + except OperationalError: + g.ubsession.rollback() return make_response(' ', 200) @@ -537,7 +543,10 @@ def HandleTagAddItem(tag_id): log.debug("Received request to add an unknown book to a collection. Silently ignoring item.") g.ubsession.merge(shelf) - g.ubsession.commit() + try: + g.ubsession.commit() + except OperationalError: + g.ubsession.rollback() return make_response('', 201) @@ -578,7 +587,10 @@ def HandleTagRemoveItem(tag_id): shelf.books.filter(ub.BookShelf.book_id == book.id).delete() except KeyError: items_unknown_to_calibre.append(item) - g.ubsession.commit() + try: + g.ubsession.commit() + except OperationalError: + g.ubsession.rollback() if items_unknown_to_calibre: log.debug("Received request to remove an unknown book to a collecition. Silently ignoring item.") @@ -624,7 +636,10 @@ def sync_shelves(sync_token, sync_results): "ChangedTag": tag }) sync_token.tags_last_modified = new_tags_last_modified - g.ubsession.commit() + try: + g.ubsession.commit() + except OperationalError: + g.ubsession.rollback() # Creates a Kobo "Tag" object from a ub.Shelf object @@ -705,7 +720,10 @@ def HandleStateRequest(book_uuid): abort(400, description="Malformed request data is missing 'ReadingStates' key") g.ubsession.merge(kobo_reading_state) - g.ubsession.commit() + try: + g.ubsession.commit() + except OperationalError: + g.ubsession.rollback() return jsonify({ "RequestResult": "Success", "UpdateResults": [update_results_response], @@ -743,7 +761,10 @@ def get_or_create_reading_state(book_id): kobo_reading_state.statistics = ub.KoboStatistics() book_read.kobo_reading_state = kobo_reading_state g.ubsession.add(book_read) - g.ubsession.commit() + try: + g.ubsession.commit() + except OperationalError: + g.ubsession.rollback() return book_read.kobo_reading_state @@ -846,7 +867,10 @@ def HandleBookDeletionRequest(book_uuid): archived_book.last_modified = datetime.datetime.utcnow() g.ubsession.merge(archived_book) - g.ubsession.commit() + try: + g.ubsession.commit() + except OperationalError: + g.ubsession.rollback() return ("", 204) diff --git a/cps/kobo_auth.py b/cps/kobo_auth.py index e936acce..dda5d018 100644 --- a/cps/kobo_auth.py +++ b/cps/kobo_auth.py @@ -66,6 +66,7 @@ from os import urandom from flask import g, Blueprint, url_for, abort, request from flask_login import login_user, login_required from flask_babel import gettext as _ +from sqlalchemy.exc import OperationalError from . import logger, ub, lm from .web import render_title_template @@ -147,7 +148,10 @@ def generate_auth_token(user_id): auth_token.token_type = 1 g.ubsession.add(auth_token) - g.ubsession.commit() + try: + g.ubsession.commit() + except OperationalError: + g.ubsession.rollback() return render_title_template( "generate_kobo_auth_url.html", title=_(u"Kobo Setup"), @@ -164,5 +168,8 @@ def delete_auth_token(user_id): # Invalidate any prevously generated Kobo Auth token for this user. g.ubsession.query(ub.RemoteAuthToken).filter(ub.RemoteAuthToken.user_id == user_id)\ .filter(ub.RemoteAuthToken.token_type==1).delete() - g.ubsession.commit() + try: + g.ubsession.commit() + except OperationalError: + g.ubsession.rollback() return "" diff --git a/cps/oauth_bb.py b/cps/oauth_bb.py index 1305c977..000a7376 100644 --- a/cps/oauth_bb.py +++ b/cps/oauth_bb.py @@ -32,6 +32,7 @@ from flask_dance.contrib.github import make_github_blueprint, github from flask_dance.contrib.google import make_google_blueprint, google from flask_login import login_user, current_user from sqlalchemy.orm.exc import NoResultFound +from sqlalchemy.exc import OperationalError from . import constants, logger, config, app, ub from .web import login_required @@ -104,12 +105,18 @@ if ub.oauth_support: oauthProvider.provider_name = "github" oauthProvider.active = False g.ubsession.add(oauthProvider) - g.ubsession.commit() + try: + g.ubsession.commit() + except OperationalError: + g.ubsession.rollback() oauthProvider = ub.OAuthProvider() oauthProvider.provider_name = "google" oauthProvider.active = False g.ubsession.add(oauthProvider) - g.ubsession.commit() + try: + g.ubsession.commit() + except OperationalError: + g.ubsession.rollback() oauth_ids = g.ubsession.query(ub.OAuthProvider).all() ele1 = dict(provider_name='github', diff --git a/cps/shelf.py b/cps/shelf.py index 8ab0b5d8..b312922d 100644 --- a/cps/shelf.py +++ b/cps/shelf.py @@ -322,8 +322,11 @@ def delete_shelf_helper(cur_shelf): g.ubsession.delete(cur_shelf) g.ubsession.query(ub.BookShelf).filter(ub.BookShelf.shelf == shelf_id).delete() g.ubsession.add(ub.ShelfArchive(uuid=cur_shelf.uuid, user_id=cur_shelf.user_id)) - g.ubsession.commit() - log.info("successfully deleted %s", cur_shelf) + try: + g.ubsession.commit() + log.info("successfully deleted %s", cur_shelf) + except OperationalError: + g.ubsession.rollback() diff --git a/cps/ub.py b/cps/ub.py index 9a7069dc..a1e67655 100644 --- a/cps/ub.py +++ b/cps/ub.py @@ -681,13 +681,19 @@ def update_download(book_id, user_id): if not check: new_download = Downloads(user_id=user_id, book_id=book_id) g.ubsession.add(new_download) - g.ubsession.commit() + try: + g.ubsession.commit() + except exc.OperationalError: + g.ubsession.rollback() # Delete non exisiting downloaded books in calibre-web's own database def delete_download(book_id): g.ubsession.query(Downloads).filter(book_id == Downloads.book_id).delete() - g.ubsession.commit() + try: + g.ubsession.commit() + except exc.OperationalError: + g.ubsession.rollback() # Generate user Guest (translated text), as anonymous user, no rights def create_anonymous_user(session): @@ -746,7 +752,7 @@ def init_db(app_db_path): session.close() -def dispose(): +'''def dispose(): global session old_session = session @@ -760,4 +766,4 @@ def dispose(): try: old_session.bind.dispose() except Exception: - pass + pass''' diff --git a/cps/web.py b/cps/web.py index 15d22d60..e93a5af8 100644 --- a/cps/web.py +++ b/cps/web.py @@ -434,7 +434,10 @@ def bookmark(book_id, book_format): ub.Bookmark.book_id == book_id, ub.Bookmark.format == book_format)).delete() if not bookmark_key: - g.ubsession.commit() + try: + g.ubsession.commit() + except OperationalError: + g.ubsession.rollback() return "", 204 lbookmark = ub.Bookmark(user_id=current_user.id, @@ -442,7 +445,10 @@ def bookmark(book_id, book_format): format=book_format, bookmark_key=bookmark_key) g.ubsession.merge(lbookmark) - g.ubsession.commit() + try: + g.ubsession.commit() + except OperationalError: + g.ubsession.rollback() return "", 201 @@ -467,7 +473,10 @@ def toggle_read(book_id): kobo_reading_state.statistics = ub.KoboStatistics() book.kobo_reading_state = kobo_reading_state g.ubsession.merge(book) - g.ubsession.commit() + try: + g.ubsession.commit() + except OperationalError: + g.ubsession.rollback() else: try: calibre_db.update_title_sort(config) @@ -501,7 +510,10 @@ def toggle_archived(book_id): archived_book = ub.ArchivedBook(user_id=current_user.id, book_id=book_id) archived_book.is_archived = True g.ubsession.merge(archived_book) - g.ubsession.commit() + try: + g.ubsession.commit() + except OperationalError: + g.ubsession.rollback() return "" @@ -1086,11 +1098,12 @@ def update_table_settings(): except AttributeError: pass g.ubsession.commit() - except InvalidRequestError: + except (InvalidRequestError, OperationalError): log.error("Invalid request received: %r ", request, ) return "Invalid request", 400 return "" + @web.route("/author") @login_required_if_no_ano def author_list(): @@ -1676,7 +1689,10 @@ def logout(): def remote_login(): auth_token = ub.RemoteAuthToken() g.ubsession.add(auth_token) - g.ubsession.commit() + try: + g.ubsession.commit() + except OperationalError: + g.ubsession.rollback() verify_url = url_for('web.verify_token', token=auth_token.auth_token, _external=true) log.debug(u"Remot Login request with token: %s", auth_token.auth_token) @@ -1708,7 +1724,10 @@ def verify_token(token): # Update token with user information auth_token.user_id = current_user.id auth_token.verified = True - g.ubsession.commit() + try: + g.ubsession.commit() + except OperationalError: + g.ubsession.rollback() flash(_(u"Success! Please return to your device"), category="success") log.debug(u"Remote Login token for userid %s verified", auth_token.user_id) @@ -1731,7 +1750,10 @@ def token_verified(): # Token expired elif datetime.now() > auth_token.expiration: g.ubsession.delete(auth_token) - g.ubsession.commit() + try: + g.ubsession.commit() + except OperationalError: + g.ubsession.rollback() data['status'] = 'error' data['message'] = _(u"Token has expired") @@ -1744,7 +1766,10 @@ def token_verified(): login_user(user) g.ubsession.delete(auth_token) - g.ubsession.commit() + try: + g.ubsession.commit() + except OperationalError: + g.ubsession.rollback() data['status'] = 'success' log.debug(u"Remote Login for userid %s succeded", user.id) @@ -1836,14 +1861,11 @@ def profile(): g.ubsession.rollback() flash(_(u"Found an existing account for this e-mail address."), category="error") log.debug(u"Found an existing account for this e-mail address.") - '''return render_title_template("user_edit.html", - content=current_user, - translations=translations, - kobo_support=kobo_support, - title=_(u"%(name)s's profile", name=current_user.nickname), - page="me", - registered_oauth=local_oauth_check, - oauth_status=oauth_status)''' + except OperationalError as e: + g.ubsession.rollback() + log.error("Database error: %s", e) + flash(_(u"Database error: %(error)s.", error=e), category="error") + return render_title_template("user_edit.html", translations=translations, profile=1, From d15d252af7b2c3fcc508df97d795dc1f1b7c64e6 Mon Sep 17 00:00:00 2001 From: Ozzieisaacs Date: Tue, 8 Dec 2020 08:01:42 +0100 Subject: [PATCH 4/7] Session no longer expires on commit --- cps/db.py | 7 ++++--- cps/tasks/convert.py | 23 ++++++++++++++--------- 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/cps/db.py b/cps/db.py index 520d724b..491e95b3 100644 --- a/cps/db.py +++ b/cps/db.py @@ -426,18 +426,19 @@ class CalibreDB(): # instances alive once they reach the end of their respective scopes instances = WeakSet() - def __init__(self): + def __init__(self, expire_on_commit=False): """ Initialize a new CalibreDB session """ self.session = None if self._init: - self.initSession() + self.initSession(expire_on_commit) self.instances.add(self) - def initSession(self): + def initSession(self, expire_on_commit): self.session = self.session_factory() + self.session.expire_on_commit = expire_on_commit self.update_title_sort(self.config) @classmethod diff --git a/cps/tasks/convert.py b/cps/tasks/convert.py index 659c6e9c..f9d6dd2a 100644 --- a/cps/tasks/convert.py +++ b/cps/tasks/convert.py @@ -9,7 +9,7 @@ from shutil import copyfile from sqlalchemy.exc import SQLAlchemyError from cps.services.worker import CalibreTask -from cps import calibre_db, db +from cps import db from cps import logger, config from cps.subproc_wrapper import process_open from flask_babel import gettext as _ @@ -33,8 +33,10 @@ class TaskConvert(CalibreTask): def run(self, worker_thread): self.worker_thread = worker_thread if config.config_use_google_drive: - cur_book = calibre_db.get_book(self.bookid) - data = calibre_db.get_book_format(self.bookid, self.settings['old_book_format']) + worker_db = db.CalibreDB() + cur_book = worker_db.get_book(self.bookid) + data = worker_db.get_book_format(self.bookid, self.settings['old_book_format']) + worker_db.session.close() df = gdriveutils.getFileFromEbooksFolder(cur_book.path, data.name + "." + self.settings['old_book_format'].lower()) if df: @@ -71,7 +73,7 @@ class TaskConvert(CalibreTask): def _convert_ebook_format(self): error_message = None - local_session = db.CalibreDB().session + local_db = db.CalibreDB() file_path = self.file_path book_id = self.bookid format_old_ext = u'.' + self.settings['old_book_format'].lower() @@ -82,10 +84,11 @@ class TaskConvert(CalibreTask): # this will allow send to kindle workflow to continue to work if os.path.isfile(file_path + format_new_ext): log.info("Book id %d already converted to %s", book_id, format_new_ext) - cur_book = calibre_db.get_book(book_id) + cur_book = local_db.session.get_book(book_id) self.results['path'] = file_path self.results['title'] = cur_book.title self._handleSuccess() + local_db.session.close() return os.path.basename(file_path + format_new_ext) else: log.info("Book id %d - target format of %s does not exist. Moving forward with convert.", @@ -105,18 +108,19 @@ class TaskConvert(CalibreTask): check, error_message = self._convert_calibre(file_path, format_old_ext, format_new_ext) if check == 0: - cur_book = calibre_db.get_book(book_id) + cur_book = local_db.get_book(book_id) if os.path.isfile(file_path + format_new_ext): # self.db_queue.join() new_format = db.Data(name=cur_book.data[0].name, book_format=self.settings['new_book_format'].upper(), book=book_id, uncompressed_size=os.path.getsize(file_path + format_new_ext)) try: - local_session.merge(new_format) - local_session.commit() + local_db.session.merge(new_format) + local_db.session.commit() except SQLAlchemyError as e: - local_session.rollback() + local_db.rollback() log.error("Database error: %s", e) + local_db.session.close() return self.results['path'] = cur_book.path self.results['title'] = cur_book.title @@ -125,6 +129,7 @@ class TaskConvert(CalibreTask): return os.path.basename(file_path + format_new_ext) else: error_message = _('%(format)s format not found on disk', format=format_new_ext.upper()) + local_db.session.close() log.info("ebook converter failed with error while converting book") if not error_message: error_message = _('Ebook converter failed with unknown error') From 1a9b220ec27449b558218f335358200661d643bb Mon Sep 17 00:00:00 2001 From: Ozzieisaacs Date: Tue, 8 Dec 2020 08:04:46 +0100 Subject: [PATCH 5/7] Session no longer expires on commit (only in worker thread) --- cps/db.py | 2 +- cps/tasks/convert.py | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/cps/db.py b/cps/db.py index 491e95b3..fe89c50b 100644 --- a/cps/db.py +++ b/cps/db.py @@ -426,7 +426,7 @@ class CalibreDB(): # instances alive once they reach the end of their respective scopes instances = WeakSet() - def __init__(self, expire_on_commit=False): + def __init__(self, expire_on_commit=True): """ Initialize a new CalibreDB session """ self.session = None diff --git a/cps/tasks/convert.py b/cps/tasks/convert.py index f9d6dd2a..f2fc955c 100644 --- a/cps/tasks/convert.py +++ b/cps/tasks/convert.py @@ -33,10 +33,9 @@ class TaskConvert(CalibreTask): def run(self, worker_thread): self.worker_thread = worker_thread if config.config_use_google_drive: - worker_db = db.CalibreDB() + worker_db = db.CalibreDB(expire_on_commit=False) cur_book = worker_db.get_book(self.bookid) data = worker_db.get_book_format(self.bookid, self.settings['old_book_format']) - worker_db.session.close() df = gdriveutils.getFileFromEbooksFolder(cur_book.path, data.name + "." + self.settings['old_book_format'].lower()) if df: @@ -46,10 +45,12 @@ class TaskConvert(CalibreTask): if not os.path.exists(os.path.join(config.config_calibre_dir, cur_book.path)): os.makedirs(os.path.join(config.config_calibre_dir, cur_book.path)) df.GetContentFile(datafile) + worker_db.session.close() else: error_message = _(u"%(format)s not found on Google Drive: %(fn)s", format=self.settings['old_book_format'], fn=data.name + "." + self.settings['old_book_format'].lower()) + worker_db.session.close() return error_message filename = self._convert_ebook_format() @@ -73,7 +74,7 @@ class TaskConvert(CalibreTask): def _convert_ebook_format(self): error_message = None - local_db = db.CalibreDB() + local_db = db.CalibreDB(expire_on_commit=False) file_path = self.file_path book_id = self.bookid format_old_ext = u'.' + self.settings['old_book_format'].lower() From f677dcb1f48d3ac0292ca603c2023770a66710e6 Mon Sep 17 00:00:00 2001 From: Ozzieisaacs Date: Tue, 8 Dec 2020 08:19:18 +0100 Subject: [PATCH 6/7] Fix missing optional parameter initSession --- cps/db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cps/db.py b/cps/db.py index fe89c50b..e35417dc 100644 --- a/cps/db.py +++ b/cps/db.py @@ -436,7 +436,7 @@ class CalibreDB(): self.instances.add(self) - def initSession(self, expire_on_commit): + def initSession(self, expire_on_commit=True): self.session = self.session_factory() self.session.expire_on_commit = expire_on_commit self.update_title_sort(self.config) From 986d4c99bd5b069d7b068ad6aaa4d4a1795fb808 Mon Sep 17 00:00:00 2001 From: OzzieIsaacs Date: Tue, 8 Dec 2020 11:24:07 +0100 Subject: [PATCH 7/7] Fix oauth ub session --- cps/gdriveutils.py | 2 +- cps/oauth_bb.py | 27 +- test/Calibre-Web TestSummary_Linux.html | 13604 ++++++++++++++++++++-- 3 files changed, 12740 insertions(+), 893 deletions(-) diff --git a/cps/gdriveutils.py b/cps/gdriveutils.py index 489bc6f5..1c5dec05 100644 --- a/cps/gdriveutils.py +++ b/cps/gdriveutils.py @@ -89,7 +89,7 @@ class Singleton: except AttributeError: self._instance = self._decorated() return self._instance - except ImportError as e: + except (ImportError, NameError) as e: log.debug(e) return None diff --git a/cps/oauth_bb.py b/cps/oauth_bb.py index 000a7376..620ff10e 100644 --- a/cps/oauth_bb.py +++ b/cps/oauth_bb.py @@ -100,25 +100,26 @@ def logout_oauth_user(): if ub.oauth_support: oauthblueprints = [] - if not g.ubsession.query(ub.OAuthProvider).count(): + oauth_session = ub.Scoped_Session() + if not oauth_session.query(ub.OAuthProvider).count(): oauthProvider = ub.OAuthProvider() oauthProvider.provider_name = "github" oauthProvider.active = False - g.ubsession.add(oauthProvider) + oauth_session.add(oauthProvider) try: - g.ubsession.commit() + oauth_session.commit() except OperationalError: - g.ubsession.rollback() + oauth_session.rollback() oauthProvider = ub.OAuthProvider() oauthProvider.provider_name = "google" oauthProvider.active = False - g.ubsession.add(oauthProvider) + oauth_session.add(oauthProvider) try: - g.ubsession.commit() + oauth_session.commit() except OperationalError: - g.ubsession.rollback() + oauth_session.rollback() - oauth_ids = g.ubsession.query(ub.OAuthProvider).all() + oauth_ids = oauth_session.query(ub.OAuthProvider).all() ele1 = dict(provider_name='github', id=oauth_ids[0].id, active=oauth_ids[0].active, @@ -148,7 +149,7 @@ if ub.oauth_support: scope=element['scope'] ) element['blueprint'] = blueprint - element['blueprint'].backend = OAuthBackend(ub.OAuth, g.ubsession, str(element['id']), + element['blueprint'].backend = OAuthBackend(ub.OAuth, oauth_session, str(element['id']), user=current_user, user_required=True) app.register_blueprint(blueprint, url_prefix="/login") if element['active']: @@ -192,7 +193,7 @@ if ub.oauth_support: session[provider_id + "_oauth_token"] = token # Find this OAuth token in the database, or create it - query = g.ubsession.query(ub.OAuth).filter_by( + query = oauth_session.query(ub.OAuth).filter_by( provider=provider_id, provider_user_id=provider_user_id, ) @@ -207,11 +208,11 @@ if ub.oauth_support: token=token, ) try: - g.ubsession.add(oauth_entry) - g.ubsession.commit() + oauth_session.add(oauth_entry) + oauth_session.commit() except Exception as e: log.exception(e) - g.ubsession.rollback() + oauth_session.rollback() # Disable Flask-Dance's default behavior for saving the OAuth token # Value differrs depending on flask-dance version diff --git a/test/Calibre-Web TestSummary_Linux.html b/test/Calibre-Web TestSummary_Linux.html index e96fa1ac..4133a3ff 100644 --- a/test/Calibre-Web TestSummary_Linux.html +++ b/test/Calibre-Web TestSummary_Linux.html @@ -37,20 +37,20 @@
-

Start Time: 2020-12-02 15:51:36

+

Start Time: 2020-12-08 08:19:58

-

Stop Time: 2020-12-02 18:01:04

+

Stop Time: 2020-12-08 10:00:52

-

Duration: 1h 41 min

+

Duration: 1h 15 min

@@ -511,162 +511,730 @@ AssertionError: False is not true : BMP file is not detected - + TestEbookConvertCalibreGDrive - 6 - 6 + 12 0 0 + 12 0 - Detail + Detail - +
TestEbookConvertCalibreGDrive - test_convert_email
- PASS + +
+ ERROR +
+ + + + - + -
TestEbookConvertCalibreGDrive - test_convert_failed_and_email
+
TestEbookConvertCalibreGDrive - test_convert_email
+ + +
+ ERROR +
+ + + - PASS - + -
TestEbookConvertCalibreGDrive - test_convert_only
+
TestEbookConvertCalibreGDrive - test_convert_failed_and_email
+ + +
+ ERROR +
+ + + - PASS - + -
TestEbookConvertCalibreGDrive - test_convert_parameter
+
TestEbookConvertCalibreGDrive - test_convert_failed_and_email
+ + +
+ ERROR +
+ + + - PASS - + -
TestEbookConvertCalibreGDrive - test_email_failed
+
TestEbookConvertCalibreGDrive - test_convert_only
+ + +
+ ERROR +
+ + + - PASS - + -
TestEbookConvertCalibreGDrive - test_email_only
+
TestEbookConvertCalibreGDrive - test_convert_only
- PASS - - - + +
+ ERROR +
+ + + - PASS - + -
TestEbookConvertKepubify - test_convert_only
+
TestEbookConvertCalibreGDrive - test_convert_parameter
+ + +
+ ERROR +
+ + + - PASS - + -
TestEbookConvertKepubify - test_convert_wrong_excecutable
+
TestEbookConvertCalibreGDrive - test_convert_parameter
- PASS - - - + +
+ ERROR +
+ + + - PASS - + -
TestEbookConvertGDriveKepubify - test_convert_only
+
TestEbookConvertCalibreGDrive - test_email_failed
- FAIL + ERROR
-