Merge pull request 'Added threading to gamelist thumbnail creation' (#31) from threading_gamelist into master

Reviewed-on: https://git.odecif.net:3000/odecif/bcns-gameDistributionSystem/pulls/31

api/modules/gamelist.py

@@ -3,7 +3,9 @@ from flask import jsonify, make_response
 import modules.db_connect
 from modules.functions import get_gamelist, set_gamelist
 import modules.game
-from modules.gamelist_functions import get_thumbnails, update_thumbnails
+from modules.gamelist_functions import (
+        get_threaded_thumbnails,
+        update_threaded_thumbnails)
 import glob
 
 contentpath = modules.db_connect.contentpath()
@@ -41,14 +43,14 @@ def update_gamelist():
 # Fetch displayimage for all nfo-files
 @app.route('/gamelist/displayimage')
 def get_displayimages(update=False):
-    thumbnails = get_thumbnails()
+    thumbnails = get_threaded_thumbnails()
     if (len(thumbnails) == 0) or update:
-        thumbnails = update_thumbnails(True)
+        thumbnails = update_threaded_thumbnails(True)
     return jsonify(thumbnails)
 
 
 # Update displayimages
 @app.route('/gamelist/displayimage/update')
 def update_displayimages():
-    update_thumbnails()
+    update_threaded_thumbnails()
     return make_response("<h1>Success</h1>", 200)

api/modules/gamelist_functions.py

@@ -3,6 +3,9 @@ import glob
 import os
 import json
 from modules.functions import reduceartcv2
+import threading
+from queue import Queue
+import time
 
 contentpath = modules.db_connect.contentpath()
 nfosuffix = modules.db_connect.nfosuffix()
@@ -10,12 +13,12 @@ nfosuffix = modules.db_connect.nfosuffix()
 global nfofiles
 nfofiles = []
 
-global thumbnails
-thumbnails = []
-
 global thumbnails_update_status
 thumbnails_update_status = False
 
+global threaded_thumbnails
+threaded_thumbnails = []
+
 
 # Set nfo-files list
 def set_nfofiles(nfofiles_list):
@@ -40,27 +43,30 @@ def update_nfofiles(return_list=False):
     return True
 
 
-# Set thumbnails list
-def set_thumbnails(thumbnails_list):
-    global thumbnails
-    thumbnails = thumbnails_list
+# Set threaded_thumbnails list
+def set_threaded_thumbnails(thumbnails_list):
+    global threaded_thumbnails
+    threaded_thumbnails = thumbnails_list
     return True
 
 
-# Get thumbnails list
-def get_thumbnails():
-    global thumbnails
-    return thumbnails
+# Get threaded_thumbnails list
+def get_threaded_thumbnails():
+    global threaded_thumbnails
+    return threaded_thumbnails
 
 
-# Update thumbnails list
-def update_thumbnails(return_list=False):
-    global thumbnails_update_status
-    thumbnails_update_status = True
+# Threaded printer
+def printer(queue):
+    while True:
+        message = queue.get()
+        print(message)
+
+
+# Create thumbnails
+def create_thumbnails(nfolist, lock, queue):
+    global threaded_thumbnails
     nfos = []
-    nfolist = get_nfofiles()
-    if len(nfolist) == 0:
-        nfolist = update_nfofiles(True)
     j = 0
     for nfopath in nfolist:
         nfo = json.load(open(nfopath, 'r'))
@@ -80,12 +86,54 @@ def update_thumbnails(return_list=False):
                         current_nfo['displayimage'] = reduceartcv2(
                             artpath+art['filename'], 'thumbnail')
                     except Exception as e:
-                        print("Failing cover:", art['filename'], e)
+                        queue.put((
+                            "Failing cover:" +
+                            str(art['filename']) +
+                            str(e)))
         nfos.append(current_nfo)
-        j += 1
-        print("[", j, "/", len(nfofiles), "]", nfopath, "added")
-    set_thumbnails(nfos)
-    thumbnails_update_status = False
-    if return_list is True:
-        return nfos
+        with lock:
+            j += 1
+            queue.put((
+                "[" +
+                str(j) +
+                "/" +
+                str(len(nfolist)) +
+                "] " +
+                str(nfopath) +
+                " added"))
+    with lock:
+        set_threaded_thumbnails(get_threaded_thumbnails() + nfos)
+    return True
+
+
+# Update thumbnails list, threaded
+def update_threaded_thumbnails(return_list=False):
+    global thumbnails_update_status
+    nfolist = get_nfofiles()
+    if len(nfolist) == 0:
+        nfolist = update_nfofiles(True)
+    nl1 = nfolist[:len(nfolist)//2]
+    nl2 = nfolist[len(nfolist)//2:]
+
+    lock = threading.Lock()
+    queue = Queue()
+    printer_thread = threading.Thread(
+            target=printer,
+            args=(queue,),
+            daemon=True,
+            name="Printer")
+    printer_thread.start()
+
+    t1 = threading.Thread(target=create_thumbnails, args=(nl1, lock, queue))
+    t2 = threading.Thread(target=create_thumbnails, args=(nl2, lock, queue))
+
+    start_time = time.time()
+    t1.start()
+    t2.start()
+
+    t1.join()
+    t2.join()
+    print("Threading done! duration:", (time.time() - start_time))
+    if return_list:
+        return get_threaded_thumbnails()
     return True

site/modules/gamelist.py

@@ -60,7 +60,6 @@ def gamelist(lang_code):
 
         except Exception as e:
             print("error type:", type(e))
-
     if glist is not None:
         return render_template('gamelist.html', gamelist=glist,
                                **languages[lang_code], lang_code=lang_code, thumbnails=thumbnailslist)