From e160efbdac825dca625b4f9a9ce931af8a3bdbe5 Mon Sep 17 00:00:00 2001 From: bodybybuddha Date: Mon, 27 Aug 2018 21:27:16 -0400 Subject: [PATCH 1/7] Add GUI functions for ebookconvert any format feature --- cps/templates/book_edit.html | 20 ++++++++++++++++++-- cps/web.py | 23 +++++++++++++++++++++-- 2 files changed, 39 insertions(+), 4 deletions(-) diff --git a/cps/templates/book_edit.html b/cps/templates/book_edit.html index 319481fb..c6a965ed 100644 --- a/cps/templates/book_edit.html +++ b/cps/templates/book_edit.html @@ -124,13 +124,29 @@ {% endfor %} {% endif %} + {% if g.user.role_upload() or g.user.role_admin()%} {% if g.allow_upload %} -
- + +
+
+ + {% if display_convertbtn and conversion_formats|length > 0 %} + + + {% endif %} +
+ {% endif %} {% endif %} diff --git a/cps/web.py b/cps/web.py index 550ecfba..f56dd03d 100644 --- a/cps/web.py +++ b/cps/web.py @@ -96,7 +96,6 @@ gdrive_watch_callback_token = 'target=calibreweb-watch_files' ALLOWED_EXTENSIONS = set(['txt', 'pdf', 'epub', 'mobi', 'azw', 'azw3', 'cbr', 'cbz', 'cbt', 'djvu', 'prc', 'doc', 'docx', 'fb2']) - def md5(fname): hash_md5 = hashlib.md5() with open(fname, "rb") as f: @@ -3072,10 +3071,30 @@ def edit_book(book_id): for author in book.authors: author_names.append(author.name.replace('|', ',')) + #Option for showing convertbook button + if config.config_ebookconverter == 2: + display_convertbtn = True + else: + display_convertbtn = False + + app.logger.debug(book) + app.logger.debug(book.data) + #Determine what formats don't already exist + allowed_conversion_formats = ALLOWED_EXTENSIONS + for file in book.data: + try: + allowed_conversion_formats.remove(file.format.lower()) + except Exception: + app.logger.debug("Exception thrown:") + app.logger.debug(file.format.lower()) + + app.logger.debug(allowed_conversion_formats) + # Show form if request.method != 'POST': return render_title_template('book_edit.html', book=book, authors=author_names, cc=cc, - title=_(u"edit metadata"), page="editbook") + title=_(u"edit metadata"), page="editbook", display_convertbtn=display_convertbtn, + conversion_formats=allowed_conversion_formats) # Update book edited_books_id = set() From 4f386c8a00a68973f01894ba744e1a26e5776403 Mon Sep 17 00:00:00 2001 From: bodybybuddha Date: Tue, 28 Aug 2018 19:13:04 -0400 Subject: [PATCH 2/7] Updated GUI buttons for ebook-convert feature --- cps/helper.py | 5 ++++ cps/static/js/edit_books.js | 4 ++++ cps/templates/book_edit.html | 45 +++++++++++++++++++++++++----------- cps/web.py | 21 +++++++++++------ 4 files changed, 54 insertions(+), 21 deletions(-) diff --git a/cps/helper.py b/cps/helper.py index 5285b13e..152bcd25 100755 --- a/cps/helper.py +++ b/cps/helper.py @@ -135,6 +135,11 @@ def send_mail(book_id, kindle_mail, calibrepath, user_id): return _(u"The requested file could not be read. Maybe wrong permissions?") +# Convert existing book entry to new format +#def convert_book_format(book_id, calibrepath, new_book_format, user_id): + +# return + def get_valid_filename(value, replace_whitespace=True): """ Returns the given string converted to a string that can be used for a clean diff --git a/cps/static/js/edit_books.js b/cps/static/js/edit_books.js index e86c78e9..b4a98a80 100644 --- a/cps/static/js/edit_books.js +++ b/cps/static/js/edit_books.js @@ -254,3 +254,7 @@ $("#btn-upload-cover").on("change", function () { } // Remove c:\fake at beginning from localhost chrome $("#upload-cover").html(filename); }); + +$("#btn-book-convert").on("change", function () { + alert("woot"); +}); diff --git a/cps/templates/book_edit.html b/cps/templates/book_edit.html index abe41923..1efaf1f0 100644 --- a/cps/templates/book_edit.html +++ b/cps/templates/book_edit.html @@ -1,7 +1,7 @@ {% extends "layout.html" %} {% block body %} {% if book %} -
+
@@ -25,7 +25,37 @@
{% endif %} {% endif %} + +{% if display_convertbtn and conversion_formats|length > 0 %} +

{{_('Convert book format:')}}

+ +
+
+ + + + +
+
+ + + + +
+{% endif %} +
+
@@ -136,19 +166,6 @@
- - {% if display_convertbtn and conversion_formats|length > 0 %} - - - {% endif %} -
{% endif %} diff --git a/cps/web.py b/cps/web.py index ce05c354..9908de2f 100644 --- a/cps/web.py +++ b/cps/web.py @@ -3183,16 +3183,10 @@ def edit_book(book_id): else: display_convertbtn = False - app.logger.debug(book) - app.logger.debug(book.data) #Determine what formats don't already exist allowed_conversion_formats = ALLOWED_EXTENSIONS for file in book.data: - try: - allowed_conversion_formats.remove(file.format.lower()) - except Exception: - app.logger.debug("Exception thrown:") - app.logger.debug(file.format.lower()) + allowed_conversion_formats.remove(file.format.lower()) app.logger.debug(allowed_conversion_formats) @@ -3627,3 +3621,16 @@ def upload(): title=book.title, books_shelfs=book_in_shelfs, page="upload") return redirect(url_for("index")) + +@app.route("/admin/book/convert/", methods=['POST']) +@login_required_if_no_ano +@edit_required +def convert_bookformat(book_id): + # rtn = convert_book_format(book_id, calibrepath, new_book_format, user_id) + app.logger.debug('got here') + app.logger.debug('book id:' + str(book_id)) + app.logger.debug('from format:'+request.form['book_format_from']) + app.logger.debug('to format:' + request.form['book_format_to']) + + return redirect(url_for("index")) + From fc78d1ee072f77984cd952afc022af995649cca5 Mon Sep 17 00:00:00 2001 From: bodybybuddha Date: Tue, 28 Aug 2018 19:22:17 -0400 Subject: [PATCH 3/7] Added exception catching for file format check --- cps/web.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cps/web.py b/cps/web.py index 9908de2f..843805d4 100644 --- a/cps/web.py +++ b/cps/web.py @@ -3186,7 +3186,10 @@ def edit_book(book_id): #Determine what formats don't already exist allowed_conversion_formats = ALLOWED_EXTENSIONS for file in book.data: - allowed_conversion_formats.remove(file.format.lower()) + try: + allowed_conversion_formats.remove(file.format.lower()) + except Exception: + app.logger.warning(file.formate.lower() + ' already removed from list.') app.logger.debug(allowed_conversion_formats) From 75e6653442764991178ef897c1e5db4402d89627 Mon Sep 17 00:00:00 2001 From: bodybybuddha Date: Tue, 28 Aug 2018 22:35:24 -0400 Subject: [PATCH 4/7] #584 & #587 Convert any book format to another format --- cps/helper.py | 46 ++++++++++++++-- cps/web.py | 33 +++++++----- cps/worker.py | 141 ++++++++++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 201 insertions(+), 19 deletions(-) diff --git a/cps/helper.py b/cps/helper.py index 152bcd25..b0e38b66 100755 --- a/cps/helper.py +++ b/cps/helper.py @@ -136,9 +136,49 @@ def send_mail(book_id, kindle_mail, calibrepath, user_id): # Convert existing book entry to new format -#def convert_book_format(book_id, calibrepath, new_book_format, user_id): - -# return +def convert_book_format(book_id, calibrepath, old_book_format, new_book_format, user_id): + book = db.session.query(db.Books).filter(db.Books.id == book_id).first() + data = db.session.query(db.Data).filter(db.Data.book == book.id).filter(db.Data.format == old_book_format).first() + if not data: + error_message = _(u"%(format)s format not found for book id: %(book)d", format=old_book_format, book=book_id) + app.logger.error("convert_book_format: " + error_message) + return error_message + if ub.config.config_use_google_drive: + df = gd.getFileFromEbooksFolder(book.path, data.name + "." + old_book_format.lower()) + if df: + datafile = os.path.join(calibrepath, book.path, data.name + "." + old_book_format.lower()) + if not os.path.exists(os.path.join(calibrepath, book.path)): + os.makedirs(os.path.join(calibrepath, book.path)) + df.GetContentFile(datafile) + else: + error_message = _(u"convert_book_format: %(format)s not found on gdrive: %(fn)s", + format=old_book_format, fn=data.name + "." + old_book_format.lower()) + return error_message + file_path = os.path.join(calibrepath, book.path, data.name) + if os.path.exists(file_path + "." + old_book_format.lower()): + # append converter to queue + settings = {'old_book_format': old_book_format, + 'new_book_format': new_book_format} + + app.logger.debug("Creating worker thread:") + app.logger.debug("filepath: " + file_path + " " + + "bookid: " + str(book.id) + " " + + "userid: " + str(user_id) + " " + + "taskmsg: " + _(u"Convert to %(format)s: %(book)s", + format=new_book_format, book=book.title) + " " + + "settings:old_book_format: " + settings['old_book_format'] + " " + + "settings:new_book_format: " + settings['new_book_format'] + ) + + global_WorkerThread.add_convert_any(file_path, book.id, + user_id, _(u"Convert to %(format)s: %(book)s", + format=new_book_format, book=book.title), + settings) + return None + else: + error_message = _(u"convert_book_format: %(format)s not found: %(fn)s", + format=old_book_format, fn=data.name + "." + old_book_format.lower()) + return error_message def get_valid_filename(value, replace_whitespace=True): """ diff --git a/cps/web.py b/cps/web.py index 9908de2f..f422fd1f 100644 --- a/cps/web.py +++ b/cps/web.py @@ -3177,18 +3177,20 @@ def edit_book(book_id): for authr in book.authors: author_names.append(authr.name.replace('|', ',')) - #Option for showing convertbook button + # Option for showing convertbook button if config.config_ebookconverter == 2: display_convertbtn = True else: display_convertbtn = False - #Determine what formats don't already exist - allowed_conversion_formats = ALLOWED_EXTENSIONS - for file in book.data: - allowed_conversion_formats.remove(file.format.lower()) + # Determine what formats don't already exist + allowed_conversion_formats = ALLOWED_EXTENSIONS.copy() - app.logger.debug(allowed_conversion_formats) + for file in book.data: + try: + allowed_conversion_formats.remove(file.format.lower()) + except Exception: + app.logger.warning(file.format.lower() + ' already removed from list.') # Show form if request.method != 'POST': @@ -3626,11 +3628,18 @@ def upload(): @login_required_if_no_ano @edit_required def convert_bookformat(book_id): - # rtn = convert_book_format(book_id, calibrepath, new_book_format, user_id) - app.logger.debug('got here') - app.logger.debug('book id:' + str(book_id)) - app.logger.debug('from format:'+request.form['book_format_from']) - app.logger.debug('to format:' + request.form['book_format_to']) + app.logger.debug('converting: book id: ' + str(book_id) + + ' from: ' + request.form['book_format_from'] + + ' to: ' + request.form['book_format_to']) + rtn = helper.convert_book_format(book_id, config.config_calibre_dir, request.form['book_format_from'].upper(), + request.form['book_format_to'].upper(), current_user.nickname) + + if rtn is None: + flash(_(u"Book successfully queued for converting to %(book_format)s", + book_format=request.form['book_format_from']), + category="success") + else: + flash(_(u"There was an error sending this book: %(res)s", res=rtn), category="error") - return redirect(url_for("index")) + return redirect(url_for("get_tasks_status")) diff --git a/cps/worker.py b/cps/worker.py index e2958232..d62fedb8 100644 --- a/cps/worker.py +++ b/cps/worker.py @@ -42,6 +42,7 @@ STAT_FINISH_SUCCESS = 3 TASK_EMAIL = 1 TASK_CONVERT = 2 TASK_UPLOAD = 3 +TASK_CONVERT_ANY = 4 RET_FAIL = 0 RET_SUCCESS = 1 @@ -172,6 +173,8 @@ class WorkerThread(threading.Thread): self.send_raw_email() if self.queue[self.current]['typ'] == TASK_CONVERT: self.convert_mobi() + if self.queue[self.current]['typ'] == TASK_CONVERT_ANY: + self.convert_any_format() # TASK_UPLOAD is handled implicitly self.current += 1 else: @@ -308,7 +311,6 @@ class WorkerThread(threading.Thread): self._handleError(error_message) return - def add_convert(self, file_path, bookid, user_name, typ, settings, kindle_mail): addLock = threading.Lock() addLock.acquire() @@ -325,7 +327,6 @@ class WorkerThread(threading.Thread): self.last=len(self.queue) addLock.release() - def add_email(self, subject, filepath, attachment, settings, recipient, user_name, typ, text=_(u'This e-mail has been sent via Calibre-Web.')): # if more than 20 entries in the list, clean the list @@ -357,8 +358,140 @@ class WorkerThread(threading.Thread): self.id += 1 self.last=len(self.queue) addLock.release() - - + + def convert_any_format(self): + # convert book, and upload in case of google drive + self.queue[self.current]['status'] = STAT_STARTED + self.UIqueue[self.current]['status'] = _('Started') + self.queue[self.current]['starttime'] = datetime.now() + self.UIqueue[self.current]['formStarttime'] = self.queue[self.current]['starttime'] + filename=self.convert_ebook_format() + if web.ub.config.config_use_google_drive: + gd.updateGdriveCalibreFromLocal() + + def add_convert_any(self, file_path, bookid, user_name, typ, settings): + addLock = threading.Lock() + addLock.acquire() + if self.last >= 20: + self.delete_completed_tasks() + # progress, runtime, and status = 0 + self.id += 1 + self.queue.append({'file_path':file_path, 'bookid':bookid, 'starttime': 0, + 'status': STAT_WAITING, 'typ': TASK_CONVERT_ANY, 'settings':settings}) + self.UIqueue.append({'user': user_name, 'formStarttime': '', 'progress': " 0 %", 'type': typ, + 'runtime': '0 s', 'status': _('Waiting'),'id': self.id } ) + self.id += 1 + + self.last=len(self.queue) + addLock.release() + + def convert_ebook_format(self): + error_message = None + file_path = self.queue[self.current]['file_path'] + bookid = self.queue[self.current]['bookid'] + format_old_ext = u'.' + self.queue[self.current]['settings']['old_book_format'].lower() + format_new_ext = u'.' + self.queue[self.current]['settings']['new_book_format'].lower() + # check if converter-executable is existing + if not os.path.exists(web.ub.config.config_converterpath): + self._handleError(_(u"Convertertool %(converter)s not found", converter=web.ub.config.config_converterpath)) + return + try: + # check which converter to use kindlegen is "1" //backward compatibility assumption epub original format + if web.ub.config.config_ebookconverter == 1: + command = [web.ub.config.config_converterpath, u'"' + file_path + u'.epub"'] + else: + # Linux py2.7 encode as list without quotes no empty element for parameters + # linux py3.x no encode and as list without quotes no empty element for parameters + # windows py2.7 encode as string with quotes empty element for parameters is okay + # windows py 3.x no encode and as string with quotes empty element for parameters is okay + # separate handling for windows and linux + if os.name == 'nt': + command = web.ub.config.config_converterpath + u' "' + file_path + format_old_ext + u'" "' + \ + file_path + format_new_ext + u'" ' + web.ub.config.config_calibre + if sys.version_info < (3, 0): + command = command.encode(sys.getfilesystemencoding()) + else: + command = [web.ub.config.config_converterpath, (file_path + format_old_ext), + (file_path + format_new_ext)] + if web.ub.config.config_calibre: + command.append(web.ub.config.config_calibre) + if sys.version_info < (3, 0): + command = [ x.encode(sys.getfilesystemencoding()) for x in command ] + + p = subprocess.Popen(command, stdout=subprocess.PIPE, universal_newlines=True) + except OSError as e: + self._handleError(_(u"Ebook-converter failed: %s" % e)) + return + if web.ub.config.config_ebookconverter == 1: + nextline = p.communicate()[0] + # Format of error message (kindlegen translates its output texts): + # Error(prcgen):E23006: Language not recognized in metadata.The dc:Language field is mandatory.Aborting. + conv_error = re.search(".*\(.*\):(E\d+):\s(.*)", nextline, re.MULTILINE) + # If error occoures, store error message for logfile + if conv_error: + error_message = _(u"Kindlegen failed with Error %(error)s. Message: %(message)s", + error=conv_error.group(1), message=conv_error.group(2).strip()) + web.app.logger.debug("convert_kindlegen: " + nextline) + + else: + while p.poll() is None: + nextline = p.stdout.readline() + if os.name == 'nt' and sys.version_info < (3, 0): + nextline = nextline.decode('windows-1252') + web.app.logger.debug(nextline.strip('\r\n')) + # parse progress string from calibre-converter + progress = re.search("(\d+)%\s.*", nextline) + if progress: + self.UIqueue[self.current]['progress'] = progress.group(1) + ' %' + + #process returncode + check = p.returncode + + # kindlegen returncodes + # 0 = Info(prcgen):I1036: Mobi file built successfully + # 1 = Info(prcgen):I1037: Mobi file built with WARNINGS! + # 2 = Info(prcgen):I1038: MOBI file could not be generated because of errors! + if ( check < 2 and web.ub.config.config_ebookconverter == 1) or \ + (check == 0 and web.ub.config.config_ebookconverter == 2): + cur_book = web.db.session.query(web.db.Books).filter(web.db.Books.id == bookid).first() + + if web.ub.config.config_ebookconverter == 1: + new_format = web.db.Data(name=cur_book.data[0].name, + book_format=self.queue[self.current]['settings']['new_book_format'], + book=bookid,uncompressed_size=os.path.getsize(file_path + '.mobi')) + else: + new_format = web.db.Data(name=cur_book.data[0].name, + book_format=self.queue[self.current]['settings']['new_book_format'], + book=bookid, uncompressed_size=os.path.getsize(file_path + format_new_ext)) + + cur_book.data.append(new_format) + web.db.session.commit() + self.queue[self.current]['path'] = cur_book.path + self.queue[self.current]['title'] = cur_book.title + self.queue[self.current]['status'] = STAT_FINISH_SUCCESS + self.UIqueue[self.current]['status'] = _('Finished') + self.UIqueue[self.current]['progress'] = "100 %" + self.UIqueue[self.current]['runtime'] = self._formatRuntime( + datetime.now() - self.queue[self.current]['starttime']) + + if web.ub.config.config_ebookconverter == 1: + if web.ub.config.config_use_google_drive: + os.remove(file_path + u".epub") + + return file_path + '.mobi' + else: + if web.ub.config.config_use_google_drive: + os.remove(file_path + format_old_ext) + + return file_path + format_new_ext + + else: + web.app.logger.info("ebook converter failed with error while converting book") + if not error_message: + error_message = 'Ebook converter failed with unknown error' + self._handleError(error_message) + return + def send_raw_email(self): self.queue[self.current]['starttime'] = datetime.now() self.UIqueue[self.current]['formStarttime'] = self.queue[self.current]['starttime'] From a2dfdaad1e7f51f1d11eec1560fb7359098147f4 Mon Sep 17 00:00:00 2001 From: bodybybuddha Date: Tue, 28 Aug 2018 22:36:04 -0400 Subject: [PATCH 5/7] Revert "#584 & #587 Convert any book format to another format" This reverts commit 75e6653442764991178ef897c1e5db4402d89627. --- cps/helper.py | 46 ++-------------- cps/web.py | 33 +++++------- cps/worker.py | 141 ++------------------------------------------------ 3 files changed, 19 insertions(+), 201 deletions(-) diff --git a/cps/helper.py b/cps/helper.py index b0e38b66..152bcd25 100755 --- a/cps/helper.py +++ b/cps/helper.py @@ -136,49 +136,9 @@ def send_mail(book_id, kindle_mail, calibrepath, user_id): # Convert existing book entry to new format -def convert_book_format(book_id, calibrepath, old_book_format, new_book_format, user_id): - book = db.session.query(db.Books).filter(db.Books.id == book_id).first() - data = db.session.query(db.Data).filter(db.Data.book == book.id).filter(db.Data.format == old_book_format).first() - if not data: - error_message = _(u"%(format)s format not found for book id: %(book)d", format=old_book_format, book=book_id) - app.logger.error("convert_book_format: " + error_message) - return error_message - if ub.config.config_use_google_drive: - df = gd.getFileFromEbooksFolder(book.path, data.name + "." + old_book_format.lower()) - if df: - datafile = os.path.join(calibrepath, book.path, data.name + "." + old_book_format.lower()) - if not os.path.exists(os.path.join(calibrepath, book.path)): - os.makedirs(os.path.join(calibrepath, book.path)) - df.GetContentFile(datafile) - else: - error_message = _(u"convert_book_format: %(format)s not found on gdrive: %(fn)s", - format=old_book_format, fn=data.name + "." + old_book_format.lower()) - return error_message - file_path = os.path.join(calibrepath, book.path, data.name) - if os.path.exists(file_path + "." + old_book_format.lower()): - # append converter to queue - settings = {'old_book_format': old_book_format, - 'new_book_format': new_book_format} - - app.logger.debug("Creating worker thread:") - app.logger.debug("filepath: " + file_path + " " + - "bookid: " + str(book.id) + " " + - "userid: " + str(user_id) + " " + - "taskmsg: " + _(u"Convert to %(format)s: %(book)s", - format=new_book_format, book=book.title) + " " + - "settings:old_book_format: " + settings['old_book_format'] + " " + - "settings:new_book_format: " + settings['new_book_format'] - ) - - global_WorkerThread.add_convert_any(file_path, book.id, - user_id, _(u"Convert to %(format)s: %(book)s", - format=new_book_format, book=book.title), - settings) - return None - else: - error_message = _(u"convert_book_format: %(format)s not found: %(fn)s", - format=old_book_format, fn=data.name + "." + old_book_format.lower()) - return error_message +#def convert_book_format(book_id, calibrepath, new_book_format, user_id): + +# return def get_valid_filename(value, replace_whitespace=True): """ diff --git a/cps/web.py b/cps/web.py index f422fd1f..9908de2f 100644 --- a/cps/web.py +++ b/cps/web.py @@ -3177,20 +3177,18 @@ def edit_book(book_id): for authr in book.authors: author_names.append(authr.name.replace('|', ',')) - # Option for showing convertbook button + #Option for showing convertbook button if config.config_ebookconverter == 2: display_convertbtn = True else: display_convertbtn = False - # Determine what formats don't already exist - allowed_conversion_formats = ALLOWED_EXTENSIONS.copy() - + #Determine what formats don't already exist + allowed_conversion_formats = ALLOWED_EXTENSIONS for file in book.data: - try: - allowed_conversion_formats.remove(file.format.lower()) - except Exception: - app.logger.warning(file.format.lower() + ' already removed from list.') + allowed_conversion_formats.remove(file.format.lower()) + + app.logger.debug(allowed_conversion_formats) # Show form if request.method != 'POST': @@ -3628,18 +3626,11 @@ def upload(): @login_required_if_no_ano @edit_required def convert_bookformat(book_id): - app.logger.debug('converting: book id: ' + str(book_id) + - ' from: ' + request.form['book_format_from'] + - ' to: ' + request.form['book_format_to']) - rtn = helper.convert_book_format(book_id, config.config_calibre_dir, request.form['book_format_from'].upper(), - request.form['book_format_to'].upper(), current_user.nickname) - - if rtn is None: - flash(_(u"Book successfully queued for converting to %(book_format)s", - book_format=request.form['book_format_from']), - category="success") - else: - flash(_(u"There was an error sending this book: %(res)s", res=rtn), category="error") + # rtn = convert_book_format(book_id, calibrepath, new_book_format, user_id) + app.logger.debug('got here') + app.logger.debug('book id:' + str(book_id)) + app.logger.debug('from format:'+request.form['book_format_from']) + app.logger.debug('to format:' + request.form['book_format_to']) - return redirect(url_for("get_tasks_status")) + return redirect(url_for("index")) diff --git a/cps/worker.py b/cps/worker.py index d62fedb8..e2958232 100644 --- a/cps/worker.py +++ b/cps/worker.py @@ -42,7 +42,6 @@ STAT_FINISH_SUCCESS = 3 TASK_EMAIL = 1 TASK_CONVERT = 2 TASK_UPLOAD = 3 -TASK_CONVERT_ANY = 4 RET_FAIL = 0 RET_SUCCESS = 1 @@ -173,8 +172,6 @@ class WorkerThread(threading.Thread): self.send_raw_email() if self.queue[self.current]['typ'] == TASK_CONVERT: self.convert_mobi() - if self.queue[self.current]['typ'] == TASK_CONVERT_ANY: - self.convert_any_format() # TASK_UPLOAD is handled implicitly self.current += 1 else: @@ -311,6 +308,7 @@ class WorkerThread(threading.Thread): self._handleError(error_message) return + def add_convert(self, file_path, bookid, user_name, typ, settings, kindle_mail): addLock = threading.Lock() addLock.acquire() @@ -327,6 +325,7 @@ class WorkerThread(threading.Thread): self.last=len(self.queue) addLock.release() + def add_email(self, subject, filepath, attachment, settings, recipient, user_name, typ, text=_(u'This e-mail has been sent via Calibre-Web.')): # if more than 20 entries in the list, clean the list @@ -358,140 +357,8 @@ class WorkerThread(threading.Thread): self.id += 1 self.last=len(self.queue) addLock.release() - - def convert_any_format(self): - # convert book, and upload in case of google drive - self.queue[self.current]['status'] = STAT_STARTED - self.UIqueue[self.current]['status'] = _('Started') - self.queue[self.current]['starttime'] = datetime.now() - self.UIqueue[self.current]['formStarttime'] = self.queue[self.current]['starttime'] - filename=self.convert_ebook_format() - if web.ub.config.config_use_google_drive: - gd.updateGdriveCalibreFromLocal() - - def add_convert_any(self, file_path, bookid, user_name, typ, settings): - addLock = threading.Lock() - addLock.acquire() - if self.last >= 20: - self.delete_completed_tasks() - # progress, runtime, and status = 0 - self.id += 1 - self.queue.append({'file_path':file_path, 'bookid':bookid, 'starttime': 0, - 'status': STAT_WAITING, 'typ': TASK_CONVERT_ANY, 'settings':settings}) - self.UIqueue.append({'user': user_name, 'formStarttime': '', 'progress': " 0 %", 'type': typ, - 'runtime': '0 s', 'status': _('Waiting'),'id': self.id } ) - self.id += 1 - - self.last=len(self.queue) - addLock.release() - - def convert_ebook_format(self): - error_message = None - file_path = self.queue[self.current]['file_path'] - bookid = self.queue[self.current]['bookid'] - format_old_ext = u'.' + self.queue[self.current]['settings']['old_book_format'].lower() - format_new_ext = u'.' + self.queue[self.current]['settings']['new_book_format'].lower() - # check if converter-executable is existing - if not os.path.exists(web.ub.config.config_converterpath): - self._handleError(_(u"Convertertool %(converter)s not found", converter=web.ub.config.config_converterpath)) - return - try: - # check which converter to use kindlegen is "1" //backward compatibility assumption epub original format - if web.ub.config.config_ebookconverter == 1: - command = [web.ub.config.config_converterpath, u'"' + file_path + u'.epub"'] - else: - # Linux py2.7 encode as list without quotes no empty element for parameters - # linux py3.x no encode and as list without quotes no empty element for parameters - # windows py2.7 encode as string with quotes empty element for parameters is okay - # windows py 3.x no encode and as string with quotes empty element for parameters is okay - # separate handling for windows and linux - if os.name == 'nt': - command = web.ub.config.config_converterpath + u' "' + file_path + format_old_ext + u'" "' + \ - file_path + format_new_ext + u'" ' + web.ub.config.config_calibre - if sys.version_info < (3, 0): - command = command.encode(sys.getfilesystemencoding()) - else: - command = [web.ub.config.config_converterpath, (file_path + format_old_ext), - (file_path + format_new_ext)] - if web.ub.config.config_calibre: - command.append(web.ub.config.config_calibre) - if sys.version_info < (3, 0): - command = [ x.encode(sys.getfilesystemencoding()) for x in command ] - - p = subprocess.Popen(command, stdout=subprocess.PIPE, universal_newlines=True) - except OSError as e: - self._handleError(_(u"Ebook-converter failed: %s" % e)) - return - if web.ub.config.config_ebookconverter == 1: - nextline = p.communicate()[0] - # Format of error message (kindlegen translates its output texts): - # Error(prcgen):E23006: Language not recognized in metadata.The dc:Language field is mandatory.Aborting. - conv_error = re.search(".*\(.*\):(E\d+):\s(.*)", nextline, re.MULTILINE) - # If error occoures, store error message for logfile - if conv_error: - error_message = _(u"Kindlegen failed with Error %(error)s. Message: %(message)s", - error=conv_error.group(1), message=conv_error.group(2).strip()) - web.app.logger.debug("convert_kindlegen: " + nextline) - - else: - while p.poll() is None: - nextline = p.stdout.readline() - if os.name == 'nt' and sys.version_info < (3, 0): - nextline = nextline.decode('windows-1252') - web.app.logger.debug(nextline.strip('\r\n')) - # parse progress string from calibre-converter - progress = re.search("(\d+)%\s.*", nextline) - if progress: - self.UIqueue[self.current]['progress'] = progress.group(1) + ' %' - - #process returncode - check = p.returncode - - # kindlegen returncodes - # 0 = Info(prcgen):I1036: Mobi file built successfully - # 1 = Info(prcgen):I1037: Mobi file built with WARNINGS! - # 2 = Info(prcgen):I1038: MOBI file could not be generated because of errors! - if ( check < 2 and web.ub.config.config_ebookconverter == 1) or \ - (check == 0 and web.ub.config.config_ebookconverter == 2): - cur_book = web.db.session.query(web.db.Books).filter(web.db.Books.id == bookid).first() - - if web.ub.config.config_ebookconverter == 1: - new_format = web.db.Data(name=cur_book.data[0].name, - book_format=self.queue[self.current]['settings']['new_book_format'], - book=bookid,uncompressed_size=os.path.getsize(file_path + '.mobi')) - else: - new_format = web.db.Data(name=cur_book.data[0].name, - book_format=self.queue[self.current]['settings']['new_book_format'], - book=bookid, uncompressed_size=os.path.getsize(file_path + format_new_ext)) - - cur_book.data.append(new_format) - web.db.session.commit() - self.queue[self.current]['path'] = cur_book.path - self.queue[self.current]['title'] = cur_book.title - self.queue[self.current]['status'] = STAT_FINISH_SUCCESS - self.UIqueue[self.current]['status'] = _('Finished') - self.UIqueue[self.current]['progress'] = "100 %" - self.UIqueue[self.current]['runtime'] = self._formatRuntime( - datetime.now() - self.queue[self.current]['starttime']) - - if web.ub.config.config_ebookconverter == 1: - if web.ub.config.config_use_google_drive: - os.remove(file_path + u".epub") - - return file_path + '.mobi' - else: - if web.ub.config.config_use_google_drive: - os.remove(file_path + format_old_ext) - - return file_path + format_new_ext - - else: - web.app.logger.info("ebook converter failed with error while converting book") - if not error_message: - error_message = 'Ebook converter failed with unknown error' - self._handleError(error_message) - return - + + def send_raw_email(self): self.queue[self.current]['starttime'] = datetime.now() self.UIqueue[self.current]['formStarttime'] = self.queue[self.current]['starttime'] From 4b006fba99ad2c5fb5c3ff578aa72d2980d2ec01 Mon Sep 17 00:00:00 2001 From: bodybybuddha Date: Tue, 28 Aug 2018 23:32:30 -0400 Subject: [PATCH 6/7] Re-commit: #584 & #587 Convert any book format to another format --- cps/helper.py | 45 +++++++++++++++- cps/web.py | 28 ++++++---- cps/worker.py | 138 +++++++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 198 insertions(+), 13 deletions(-) diff --git a/cps/helper.py b/cps/helper.py index 152bcd25..e95c35c7 100755 --- a/cps/helper.py +++ b/cps/helper.py @@ -136,9 +136,50 @@ def send_mail(book_id, kindle_mail, calibrepath, user_id): # Convert existing book entry to new format -#def convert_book_format(book_id, calibrepath, new_book_format, user_id): +def convert_book_format(book_id, calibrepath, old_book_format, new_book_format, user_id): + book = db.session.query(db.Books).filter(db.Books.id == book_id).first() + data = db.session.query(db.Data).filter(db.Data.book == book.id).filter(db.Data.format == old_book_format).first() + if not data: + error_message = _(u"%(format)s format not found for book id: %(book)d", format=old_book_format, book=book_id) + app.logger.error("convert_book_format: " + error_message) + return error_message + if ub.config.config_use_google_drive: + df = gd.getFileFromEbooksFolder(book.path, data.name + "." + old_book_format.lower()) + if df: + datafile = os.path.join(calibrepath, book.path, data.name + "." + old_book_format.lower()) + if not os.path.exists(os.path.join(calibrepath, book.path)): + os.makedirs(os.path.join(calibrepath, book.path)) + df.GetContentFile(datafile) + else: + error_message = _(u"convert_book_format: %(format)s not found on gdrive: %(fn)s", + format=old_book_format, fn=data.name + "." + old_book_format.lower()) + return error_message + file_path = os.path.join(calibrepath, book.path, data.name) + if os.path.exists(file_path + "." + old_book_format.lower()): + # append converter to queue + settings = {'old_book_format': old_book_format, + 'new_book_format': new_book_format} + + app.logger.debug("Creating worker thread:") + app.logger.debug("filepath: " + file_path + " " + + "bookid: " + str(book.id) + " " + + "userid: " + str(user_id) + " " + + "taskmsg: " + _(u"Convert to %(format)s: %(book)s", + format=new_book_format, book=book.title) + " " + + "settings:old_book_format: " + settings['old_book_format'] + " " + + "settings:new_book_format: " + settings['new_book_format'] + ) + + global_WorkerThread.add_convert_any(file_path, book.id, + user_id, _(u"Convert to %(format)s: %(book)s", + format=new_book_format, book=book.title), + settings) + return None + else: + error_message = _(u"convert_book_format: %(format)s not found: %(fn)s", + format=old_book_format, fn=data.name + "." + old_book_format.lower()) + return error_message -# return def get_valid_filename(value, replace_whitespace=True): """ diff --git a/cps/web.py b/cps/web.py index 843805d4..9957b82a 100644 --- a/cps/web.py +++ b/cps/web.py @@ -3177,19 +3177,19 @@ def edit_book(book_id): for authr in book.authors: author_names.append(authr.name.replace('|', ',')) - #Option for showing convertbook button + # Option for showing convertbook button if config.config_ebookconverter == 2: display_convertbtn = True else: display_convertbtn = False - #Determine what formats don't already exist - allowed_conversion_formats = ALLOWED_EXTENSIONS + # Determine what formats don't already exist + allowed_conversion_formats = ALLOWED_EXTENSIONS.copy() for file in book.data: try: allowed_conversion_formats.remove(file.format.lower()) except Exception: - app.logger.warning(file.formate.lower() + ' already removed from list.') + app.logger.warning(file.format.lower() + ' already removed from list.') app.logger.debug(allowed_conversion_formats) @@ -3629,11 +3629,19 @@ def upload(): @login_required_if_no_ano @edit_required def convert_bookformat(book_id): - # rtn = convert_book_format(book_id, calibrepath, new_book_format, user_id) - app.logger.debug('got here') - app.logger.debug('book id:' + str(book_id)) - app.logger.debug('from format:'+request.form['book_format_from']) - app.logger.debug('to format:' + request.form['book_format_to']) + app.logger.debug('converting: book id: ' + str(book_id) + + ' from: ' + request.form['book_format_from'] + + ' to: ' + request.form['book_format_to']) + rtn = helper.convert_book_format(book_id, config.config_calibre_dir, request.form['book_format_from'].upper(), + request.form['book_format_to'].upper(), current_user.nickname) + + if rtn is None: + flash(_(u"Book successfully queued for converting to %(book_format)s", + book_format=request.form['book_format_from']), + category="success") + else: + flash(_(u"There was an error sending this book: %(res)s", res=rtn), category="error") + + return redirect(url_for("get_tasks_status")) - return redirect(url_for("index")) diff --git a/cps/worker.py b/cps/worker.py index e2958232..1d0eb6cf 100644 --- a/cps/worker.py +++ b/cps/worker.py @@ -42,6 +42,7 @@ STAT_FINISH_SUCCESS = 3 TASK_EMAIL = 1 TASK_CONVERT = 2 TASK_UPLOAD = 3 +TASK_CONVERT_ANY = 4 RET_FAIL = 0 RET_SUCCESS = 1 @@ -172,6 +173,8 @@ class WorkerThread(threading.Thread): self.send_raw_email() if self.queue[self.current]['typ'] == TASK_CONVERT: self.convert_mobi() + if self.queue[self.current]['typ'] == TASK_CONVERT_ANY: + self.convert_any_format() # TASK_UPLOAD is handled implicitly self.current += 1 else: @@ -357,7 +360,140 @@ class WorkerThread(threading.Thread): self.id += 1 self.last=len(self.queue) addLock.release() - + + def convert_any_format(self): + # convert book, and upload in case of google drive + self.queue[self.current]['status'] = STAT_STARTED + self.UIqueue[self.current]['status'] = _('Started') + self.queue[self.current]['starttime'] = datetime.now() + self.UIqueue[self.current]['formStarttime'] = self.queue[self.current]['starttime'] + filename=self.convert_ebook_format() + if web.ub.config.config_use_google_drive: + gd.updateGdriveCalibreFromLocal() + + def add_convert_any(self, file_path, bookid, user_name, typ, settings): + addLock = threading.Lock() + addLock.acquire() + if self.last >= 20: + self.delete_completed_tasks() + # progress, runtime, and status = 0 + self.id += 1 + self.queue.append({'file_path':file_path, 'bookid':bookid, 'starttime': 0, + 'status': STAT_WAITING, 'typ': TASK_CONVERT_ANY, 'settings':settings}) + self.UIqueue.append({'user': user_name, 'formStarttime': '', 'progress': " 0 %", 'type': typ, + 'runtime': '0 s', 'status': _('Waiting'),'id': self.id } ) + self.id += 1 + + self.last=len(self.queue) + addLock.release() + + def convert_ebook_format(self): + error_message = None + file_path = self.queue[self.current]['file_path'] + bookid = self.queue[self.current]['bookid'] + format_old_ext = u'.' + self.queue[self.current]['settings']['old_book_format'].lower() + format_new_ext = u'.' + self.queue[self.current]['settings']['new_book_format'].lower() + # check if converter-executable is existing + if not os.path.exists(web.ub.config.config_converterpath): + self._handleError(_(u"Convertertool %(converter)s not found", converter=web.ub.config.config_converterpath)) + return + try: + # check which converter to use kindlegen is "1" //backward compatibility assumption epub original format + if web.ub.config.config_ebookconverter == 1: + command = [web.ub.config.config_converterpath, u'"' + file_path + u'.epub"'] + else: + # Linux py2.7 encode as list without quotes no empty element for parameters + # linux py3.x no encode and as list without quotes no empty element for parameters + # windows py2.7 encode as string with quotes empty element for parameters is okay + # windows py 3.x no encode and as string with quotes empty element for parameters is okay + # separate handling for windows and linux + if os.name == 'nt': + command = web.ub.config.config_converterpath + u' "' + file_path + format_old_ext + u'" "' + \ + file_path + format_new_ext + u'" ' + web.ub.config.config_calibre + if sys.version_info < (3, 0): + command = command.encode(sys.getfilesystemencoding()) + else: + command = [web.ub.config.config_converterpath, (file_path + format_old_ext), + (file_path + format_new_ext)] + if web.ub.config.config_calibre: + command.append(web.ub.config.config_calibre) + if sys.version_info < (3, 0): + command = [ x.encode(sys.getfilesystemencoding()) for x in command ] + + p = subprocess.Popen(command, stdout=subprocess.PIPE, universal_newlines=True) + except OSError as e: + self._handleError(_(u"Ebook-converter failed: %s" % e)) + return + if web.ub.config.config_ebookconverter == 1: + nextline = p.communicate()[0] + # Format of error message (kindlegen translates its output texts): + # Error(prcgen):E23006: Language not recognized in metadata.The dc:Language field is mandatory.Aborting. + conv_error = re.search(".*\(.*\):(E\d+):\s(.*)", nextline, re.MULTILINE) + # If error occoures, store error message for logfile + if conv_error: + error_message = _(u"Kindlegen failed with Error %(error)s. Message: %(message)s", + error=conv_error.group(1), message=conv_error.group(2).strip()) + web.app.logger.debug("convert_kindlegen: " + nextline) + + else: + while p.poll() is None: + nextline = p.stdout.readline() + if os.name == 'nt' and sys.version_info < (3, 0): + nextline = nextline.decode('windows-1252') + web.app.logger.debug(nextline.strip('\r\n')) + # parse progress string from calibre-converter + progress = re.search("(\d+)%\s.*", nextline) + if progress: + self.UIqueue[self.current]['progress'] = progress.group(1) + ' %' + + #process returncode + check = p.returncode + + # kindlegen returncodes + # 0 = Info(prcgen):I1036: Mobi file built successfully + # 1 = Info(prcgen):I1037: Mobi file built with WARNINGS! + # 2 = Info(prcgen):I1038: MOBI file could not be generated because of errors! + if ( check < 2 and web.ub.config.config_ebookconverter == 1) or \ + (check == 0 and web.ub.config.config_ebookconverter == 2): + cur_book = web.db.session.query(web.db.Books).filter(web.db.Books.id == bookid).first() + + if web.ub.config.config_ebookconverter == 1: + new_format = web.db.Data(name=cur_book.data[0].name, + book_format=self.queue[self.current]['settings']['new_book_format'], + book=bookid,uncompressed_size=os.path.getsize(file_path + '.mobi')) + else: + new_format = web.db.Data(name=cur_book.data[0].name, + book_format=self.queue[self.current]['settings']['new_book_format'], + book=bookid, uncompressed_size=os.path.getsize(file_path + format_new_ext)) + + cur_book.data.append(new_format) + web.db.session.commit() + self.queue[self.current]['path'] = cur_book.path + self.queue[self.current]['title'] = cur_book.title + self.queue[self.current]['status'] = STAT_FINISH_SUCCESS + self.UIqueue[self.current]['status'] = _('Finished') + self.UIqueue[self.current]['progress'] = "100 %" + self.UIqueue[self.current]['runtime'] = self._formatRuntime( + datetime.now() - self.queue[self.current]['starttime']) + + if web.ub.config.config_ebookconverter == 1: + if web.ub.config.config_use_google_drive: + os.remove(file_path + u".epub") + + return file_path + '.mobi' + else: + if web.ub.config.config_use_google_drive: + os.remove(file_path + format_old_ext) + + return file_path + format_new_ext + + else: + web.app.logger.info("ebook converter failed with error while converting book") + if not error_message: + error_message = 'Ebook converter failed with unknown error' + self._handleError(error_message) + return + def send_raw_email(self): self.queue[self.current]['starttime'] = datetime.now() From 22ecdfd2bcebb306ee700096650848f04fab0333 Mon Sep 17 00:00:00 2001 From: bodybybuddha Date: Wed, 29 Aug 2018 00:49:50 -0400 Subject: [PATCH 7/7] Refactor edit_books.js for unused code; re-add glyphicon to convert button; Add form validation for convert button --- cps/static/js/edit_books.js | 3 --- cps/templates/book_edit.html | 2 +- cps/web.py | 8 ++++++++ 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/cps/static/js/edit_books.js b/cps/static/js/edit_books.js index b4a98a80..1d182887 100644 --- a/cps/static/js/edit_books.js +++ b/cps/static/js/edit_books.js @@ -255,6 +255,3 @@ $("#btn-upload-cover").on("change", function () { $("#upload-cover").html(filename); }); -$("#btn-book-convert").on("change", function () { - alert("woot"); -}); diff --git a/cps/templates/book_edit.html b/cps/templates/book_edit.html index 1efaf1f0..6a6173e9 100644 --- a/cps/templates/book_edit.html +++ b/cps/templates/book_edit.html @@ -48,7 +48,7 @@
- + diff --git a/cps/web.py b/cps/web.py index 9957b82a..c44e456b 100644 --- a/cps/web.py +++ b/cps/web.py @@ -3191,6 +3191,7 @@ def edit_book(book_id): except Exception: app.logger.warning(file.format.lower() + ' already removed from list.') + app.logger.debug('Allowed conversion formats:') app.logger.debug(allowed_conversion_formats) # Show form @@ -3629,6 +3630,13 @@ def upload(): @login_required_if_no_ano @edit_required def convert_bookformat(book_id): + # check to see if we have form fields to work with - if not send user back + book_format_from = request.form.get('book_format_from', None) + book_format_to = request.form.get('book_format_to', None) + + if (book_format_from is None) or (book_format_to is None): + return redirect(request.environ["HTTP_REFERER"]) + app.logger.debug('converting: book id: ' + str(book_id) + ' from: ' + request.form['book_format_from'] + ' to: ' + request.form['book_format_to'])