diff --git a/.vscode/settings.json b/.vscode/settings.json index b023055..c4a3a95 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,5 +1,5 @@ { - "python.pythonPath": "/usr/bin/python3", + "python.pythonPath": "/usr/bin/python3.5", "python.linting.pylintEnabled": true, "python.linting.pep8Enabled": false, "python.linting.enabled": true diff --git a/Multiprocessing test/__pycache__/globvar.cpython-35.pyc b/Multiprocessing test/__pycache__/globvar.cpython-35.pyc new file mode 100644 index 0000000..d0b98cf Binary files /dev/null and b/Multiprocessing test/__pycache__/globvar.cpython-35.pyc differ diff --git a/Multiprocessing test/globvar.py b/Multiprocessing test/globvar.py new file mode 100644 index 0000000..216682b --- /dev/null +++ b/Multiprocessing test/globvar.py @@ -0,0 +1,56 @@ +# You are free to use and/or change this code for +# your own needs. + +# Original code (c)2018 Jan Lerking +# Program to convert C-header (*.h) files to nasm include files (*.inc), +# for direct usage in assembly programming using nasm/yasm. + +from pathlib import Path +import multiprocessing + +def init(): + global ticker_num + ticker_num = 1000 + global home + home = str(Path.home()) + global num_cores + num_cores = multiprocessing.cpu_count() + global queue + queue = 0 + global fraction + fraction = 0 + global worker + worker = [] + global listener + listener = [] + global thread + thread = [] + global tupline + tupline = [] + global preproc + preproc = () + global filelist + filelist = [] + global folderlist + folderlist = [] + global cnt + cnt = 0 + global srcdir + srcdir = '' + global destdir + destdir = '' + global fileindex + fileindex = 0 + global filecnt + filecnt = 0 + global incinc + incinc = '' + global defdir + defdir = False + global defsrc + defsrc = '/usr/include' + global defdest + defdest = home+'/include' + global count + count = 0 + diff --git a/Multiprocessing test/ticker.glade b/Multiprocessing test/ticker.glade new file mode 100644 index 0000000..9720883 --- /dev/null +++ b/Multiprocessing test/ticker.glade @@ -0,0 +1,398 @@ + + + + + + False + False + + + True + False + vertical + + + gtk-execute + True + True + True + True + 0.5899999737739563 + True + + + + False + False + 0 + + + + + True + False + end + True + True + immediate + vertical + + + Ticker + True + False + 5 + 5 + 5 + 5 + True + True + 0 + + + True + False + immediate + 12 + + + True + False + True + + + True + False + end + baseline + 5 + Items put in Queue: + right + end + + + 0 + 0 + + + + + True + False + start + True + + + 1 + 0 + + + + + True + False + 5 + 5 + 5 + 5 + + + 0 + 1 + 3 + + + + + True + False + end + 5 + Items currently in Queue: + right + + + 0 + 2 + + + + + True + False + start + True + + + 1 + 2 + + + + + True + False + 5 + 5 + 5 + 5 + + + 0 + 3 + 3 + + + + + True + False + end + 5 + Items processed in Queue: + right + + + 0 + 4 + + + + + True + False + start + True + + + 1 + 4 + + + + + True + False + 5 + 5 + 5 + 5 + True + + + 0 + 5 + 3 + + + + + + + + + + + + + + + + + + True + False + 5 + Ticker Queue + + + + + False + False + 0 + + + + + True + False + 5 + 5 + 5 + 5 + True + True + 0 + + + True + False + 12 + + + True + False + + + True + False + end + 5 + Items put in Queue: + right + end + + + 0 + 0 + + + + + True + False + start + True + + + 1 + 0 + + + + + True + False + 5 + 5 + 5 + 5 + True + + + 0 + 1 + 3 + + + + + True + False + end + 5 + Items currently in Queue: + + + 0 + 2 + + + + + True + False + start + True + + + 1 + 2 + + + + + True + False + 5 + 5 + 5 + 5 + + + 0 + 3 + 3 + + + + + True + False + end + 5 + Items processed in Queue: + + + 0 + 4 + + + + + True + False + start + True + + + 1 + 4 + + + + + True + False + 5 + 5 + 5 + 5 + True + + + 0 + 5 + 3 + + + + + + + + + + + + + + + + + + True + False + 5 + Work Queue + + + + + False + False + 1 + + + + + False + False + 1 + + + + + + diff --git a/Multiprocessing test/ticker.glade~ b/Multiprocessing test/ticker.glade~ new file mode 100644 index 0000000..61a3375 --- /dev/null +++ b/Multiprocessing test/ticker.glade~ @@ -0,0 +1,397 @@ + + + + + + False + False + + + True + False + vertical + + + gtk-execute + True + True + True + True + 0.5899999737739563 + True + + + + False + False + 0 + + + + + True + False + end + True + True + immediate + vertical + + + Ticker + True + False + 5 + 5 + 5 + 5 + True + True + 0 + + + True + False + immediate + 12 + + + True + False + + + True + False + end + baseline + 5 + Items put in Queue: + right + end + + + 0 + 0 + + + + + True + False + start + True + + + 1 + 0 + + + + + True + False + 5 + 5 + 5 + 5 + + + 0 + 1 + 3 + + + + + True + False + end + 5 + Items currently in Queue: + right + + + 0 + 2 + + + + + True + False + start + True + + + 1 + 2 + + + + + True + False + 5 + 5 + 5 + 5 + + + 0 + 3 + 3 + + + + + True + False + end + 5 + Items processed in Queue: + right + + + 0 + 4 + + + + + True + False + start + True + + + 1 + 4 + + + + + True + False + 5 + 5 + 5 + 5 + True + + + 0 + 5 + 3 + + + + + + + + + + + + + + + + + + True + False + 5 + Ticker Queue + + + + + False + False + 0 + + + + + True + False + 5 + 5 + 5 + 5 + True + True + 0 + + + True + False + 12 + + + True + False + + + True + False + end + 5 + Items put in Queue: + right + end + + + 0 + 0 + + + + + True + False + start + True + + + 1 + 0 + + + + + True + False + 5 + 5 + 5 + 5 + True + + + 0 + 1 + 3 + + + + + True + False + end + 5 + Items currently in Queue: + + + 0 + 2 + + + + + True + False + start + True + + + 1 + 2 + + + + + True + False + 5 + 5 + 5 + 5 + + + 0 + 3 + 3 + + + + + True + False + end + 5 + Items processed in Queue: + + + 0 + 4 + + + + + True + False + start + True + + + 1 + 4 + + + + + True + False + 5 + 5 + 5 + 5 + True + + + 0 + 5 + 3 + + + + + + + + + + + + + + + + + + True + False + 5 + Work Queue + + + + + False + False + 1 + + + + + False + False + 1 + + + + + + diff --git a/Multiprocessing test/ticker_gtk.py b/Multiprocessing test/ticker_gtk.py new file mode 100755 index 0000000..a54dcfa --- /dev/null +++ b/Multiprocessing test/ticker_gtk.py @@ -0,0 +1,281 @@ +#!/usr/bin/env python3.5 + +# You are free to use and/or change this code for +# your own needs. + +# Original code (c)2018 Jan Lerking +# Program to convert C-header (*.h) files to nasm include files (*.inc), +# for direct usage in assembly programming using nasm/yasm. + +import sys +import gi +gi.require_version('Gtk', '3.0') +from gi.repository import Gtk, Gio, GObject as gobject +import multiprocessing +import threading +import globvar +import time +#from h2inc_mp import start_workers +#from h2inc_fp import sourcedir_filecnt, sourcedir_foldercnt + +globvar.init() + + +gobject.threads_init() + +class Listener(gobject.GObject): + __gsignals__ = { + 'ticker_update' : (gobject.SIGNAL_RUN_LAST, + gobject.TYPE_NONE, + (gobject.TYPE_FLOAT, + gobject.TYPE_FLOAT, + gobject.TYPE_FLOAT)), + 'ticker_finished' : (gobject.SIGNAL_RUN_LAST, + gobject.TYPE_NONE, + ()), + 'handlers_finished' : (gobject.SIGNAL_RUN_LAST, + gobject.TYPE_NONE, + ()), + 'workers_update' : (gobject.SIGNAL_RUN_LAST, + gobject.TYPE_NONE, + (gobject.TYPE_FLOAT, + gobject.TYPE_FLOAT, + gobject.TYPE_FLOAT)), + 'workers_finished' : (gobject.SIGNAL_RUN_LAST, + gobject.TYPE_NONE, + ()) + } + + def __init__(self, queue): + gobject.GObject.__init__(self) + self.signal_queue = queue + + def go(self): + print("Listener has started") + while True: + # Listen for results on the queue and process them accordingly + data = self.signal_queue.get() + print(data) + # Check if finished + if data[0]=="t_finished": + print("Ticker is finishing.") + self.emit("ticker_finished", data[0]) + return + elif data[0]=="ticker_update": + print("Ticker is working.") + self.emit("update_ticker") + return + elif data[0]=="work_update": + print("Workers are working.") + self.emit("update_workers", data[0], data[1], data[2]) + return + else: + self.emit('update_ticker', data[0], data[1], data[2]) + +gobject.type_register(Listener) + +class Worker(): + def __init__(self, sq, wq, i): + self.signal_queue = sq + self.work_queue = wq + self.iterations = i + + def go(self): + while True: + task = self.work_queue.get(2) + lock = multiprocessing.Lock().acquire() + tasks = self.work_queue.qsize() + lock.release() + with self.iterations.get_lock(): + if task is None: + break + self.iterations.value += 1 + tasks = tasks/1000 + time.sleep(0.001) + self.work_queue.task_done() + self.signal_queue.put("work_update", tasks, self.iterations) + self.work_queue.task_done() + self.signal_queue.put("work_finished") + +class Handler(): + def __init__(self, sq, tq, wq, i): + self.signal_queue = sq + self.ticker_queue = tq + self.work_queue = wq + self.iterations = i + + def go(self): + while self.ticker_queue.empty() is not True: + ticker = self.ticker_queue.get(0.1) + self.work_queue.put(ticker-1000) + time.sleep(0.01) + self.ticker_queue.task_done() + +#def looprun(n): + #for i in tickers: + #ticker_queue.put(i) + + #for i in range(20): + #t = threading.Thread(target = handler, args = (ticker_queue, work_queue)) + #t.deamon = True + #t.start() + #threads.append(t) + + #for i in range(8): + #p = multiprocessing.Process(target = worker, args = (work_queue, iterations, )) + #p.deamon = True + #p.start() + #processes.append(p) + + #ticker_queue.join() + #for i in range(8): + #work_queue.put(None) + + #work_queue.join() + + #print("Closing down workers") + #for p in processes: + #p.join() + #print("Run:",n) + #print("Total number of iterations:", iterations.value) + #return + +#for n in range(100): + #looprun(n+1) + +class ticker: + + global app + + def __init__(self): + + self.ticker_put_count = 0 + self.ticker_current_count = 0 + self.ticker_processed_count = 0 + self.work_put_count = 0 + self.work_current_count = 0 + self.work_processed_count = 0 + self.ticker_num = globvar.ticker_num + self.fraction = 0 + self.process = None + self.signal_queue = multiprocessing.JoinableQueue() + self.ticker_queue = multiprocessing.JoinableQueue() + self.work_queue = multiprocessing.JoinableQueue() + self.iterations = multiprocessing.Value('i', 0) + self.tickers = range(globvar.ticker_num) + self.work_num = 0 + self.processes = [] + self.threads = [] + self.app = Gtk.Application.new("org.ticker", Gio.ApplicationFlags(0)) + self.app.connect("activate", self.on_app_activate) + self.app.connect("shutdown", self.on_app_shutdown) + app = self.app + + def on_app_activate(self, app): + + builder = Gtk.Builder() + builder.add_from_file("ticker.glade") + builder.connect_signals(self) + + self.obj = builder.get_object + self.obj("window").set_application(app) + self.obj("window").set_wmclass("ticker","ticker") + self.obj("window").set_title("ticker - v.0.0.1") + self.obj("window").show_all() + self.obj("ticker_put_count_label").set_text(str(self.ticker_put_count)) + self.obj("ticker_items_count_label").set_text(str(self.ticker_current_count)) + self.obj("ticker_processed_count_label").set_text(str(self.ticker_processed_count)) + self.obj("work_put_count_label").set_text(str(self.work_put_count)) + self.obj("work_items_count_label").set_text(str(self.work_current_count)) + self.obj("work_processed_count_label").set_text(str(self.work_processed_count)) + + def on_app_shutdown(self, app): + self.app.quit() + + def run(self, argv): + self.app.run(argv) + + def update_ticker(self, obj, index1, index2, index3, data=None): + self.val_1 = index1*self.fraction + self.val_2 = index2*self.fraction + self.val_3 = index3*self.fraction + self.obj("ticker_put_count_label").set_text(index1) + self.obj("ticker_put_progressbar").set_fraction(self.val_1) + self.obj("ticker_items_count_label").set_text(index2) + self.obj("ticker_items_progressbar").set_fraction(self.val_2) + self.obj("ticker_processed_count_label").set_text(index3) + self.obj("ticker_processed_progressbar").set_fraction(self.val_3) + + def update_work(self, obj, index1, index2, index3, data=None): + self.val_1 = index1*self.fraction + self.val_2 = index2*self.fraction + self.val_3 = index3*self.fraction + self.obj("work_put_count_label").set_text(index1) + self.obj("work_put_progressbar").set_fraction(self.val_1) + self.obj("work_items_count_label").set_text(index2) + self.obj("work_items_progressbar").set_fraction(self.val_2) + self.obj("work_processed_count_label").set_text(index3) + self.obj("work_processed_progressbar").set_fraction(self.val_3) + + def ticker_Finished(self, obj, data=None): + for t in range(20): + t = threading.Thread(target = Handler.go, args = (self.ticker_queue, self.work_queue)) + t.deamon = True + t.start() + self.threads.append(t) + + self.obj("ticker_put_count_label").set_text(self.ticker_num) + self.obj("ticker_put_progressbar").set_fraction(1.0) + self.obj("ticker_items_count_label").set_text("0") + self.obj("ticker_items_progressbar").set_fraction(0.0) + self.obj("ticker_processed_count_label").set_text(self.ticker_num) + self.obj("ticker_processed_progressbar").set_fraction(1.0) + + def workers_Finished(self, obj, data=None): + if self.process==None: + raise RuntimeError("No worker process started") + print("all done; joining worker process") + self.process.join() + self.process = None + + self.obj("work_put_count_label").set_text(self.work_num) + self.obj("work_put_progressbar").set_fraction(1.0) + self.obj("work_items_count_label").set_text("0") + self.obj("work_items_progressbar").set_fraction(0.0) + self.obj("work_processed_count_label").set_text(self.work_num) + self.obj("work_processed_progressbar").set_fraction(1.0) + + def on_window_destroy(self,window): + window.close() + + def on_execute_clicked(self,widget): + print("Creating Listener") + listener = Listener(self.signal_queue) + listener.connect("ticker_update",self.update_ticker) + listener.connect("ticker_finished",self.ticker_Finished) + listener.connect("workers_update",self.update_work) + listener.connect("workers_finished",self.workers_Finished) + + print("Starting Listener") + thread = threading.Thread(target=listener.go, args=()) + thread.start() + + for i in self.tickers: + self.ticker_queue.put(i) + + for i in range(8): + p = multiprocessing.Process(target = Worker.go, args = (self.work_queue, self.iterations, )) + p.deamon = True + p.start() + self.processes.append(p) + + self.ticker_queue.join() + for i in range(8): + self.work_queue.put(None) + + + +app = ticker() +app.run(sys.argv) + + diff --git a/Multiprocessing test/ticker_new.py b/Multiprocessing test/ticker_new.py index 4c29959..9c2e732 100644 --- a/Multiprocessing test/ticker_new.py +++ b/Multiprocessing test/ticker_new.py @@ -58,8 +58,8 @@ def looprun(n): print("Total number of iterations:", iterations.value) return -for n in range(1): - looprun(n) +for n in range(100): + looprun(n+1)