diff --git a/.vscode/settings.json b/.vscode/settings.json index c4a3a95..5918f79 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,5 +1,5 @@ { - "python.pythonPath": "/usr/bin/python3.5", + "python.pythonPath": "/home/jan/anaconda3/bin/python", "python.linting.pylintEnabled": true, "python.linting.pep8Enabled": false, "python.linting.enabled": true diff --git a/Multiprocessing test/ticker.glade b/Multiprocessing test/ticker.glade index 9720883..2b8d485 100644 --- a/Multiprocessing test/ticker.glade +++ b/Multiprocessing test/ticker.glade @@ -3,7 +3,9 @@ + 300 False + 0.97999999999999998 False @@ -63,7 +65,7 @@ True False - end + start baseline 5 Items put in Queue: @@ -80,7 +82,6 @@ True False start - True 1 @@ -106,7 +107,7 @@ True False - end + start 5 Items currently in Queue: right @@ -121,7 +122,6 @@ True False start - True 1 @@ -147,7 +147,7 @@ True False - end + start 5 Items processed in Queue: right @@ -162,7 +162,6 @@ True False start - True 1 @@ -237,7 +236,7 @@ True False - end + start 5 Items put in Queue: right @@ -253,7 +252,6 @@ True False start - True 1 @@ -280,7 +278,7 @@ True False - end + start 5 Items currently in Queue: @@ -294,7 +292,6 @@ True False start - True 1 @@ -320,7 +317,7 @@ True False - end + start 5 Items processed in Queue: @@ -334,7 +331,6 @@ True False start - True 1 diff --git a/Multiprocessing test/ticker.glade~ b/Multiprocessing test/ticker.glade~ index 61a3375..1d1698f 100644 --- a/Multiprocessing test/ticker.glade~ +++ b/Multiprocessing test/ticker.glade~ @@ -58,6 +58,7 @@ True False + True True @@ -79,7 +80,6 @@ True False start - True 1 @@ -120,7 +120,6 @@ True False start - True 1 @@ -161,7 +160,6 @@ True False start - True 1 @@ -252,7 +250,6 @@ True False start - True 1 @@ -293,7 +290,6 @@ True False start - True 1 @@ -333,7 +329,6 @@ True False start - True 1 diff --git a/Multiprocessing test/ticker_gtk.py b/Multiprocessing test/ticker_gtk.py index 76025b7..e9cbe56 100755 --- a/Multiprocessing test/ticker_gtk.py +++ b/Multiprocessing test/ticker_gtk.py @@ -25,18 +25,20 @@ gobject.threads_init() class Listener(gobject.GObject): __gsignals__ = { + 'ticker_update' : (gobject.SIGNAL_RUN_LAST, + gobject.TYPE_NONE, + (gobject.TYPE_INT,)), 'handlers_update' : (gobject.SIGNAL_RUN_LAST, gobject.TYPE_NONE, (gobject.TYPE_FLOAT, gobject.TYPE_FLOAT, - gobject.TYPE_FLOAT)), + gobject.TYPE_FLOAT,)), 'handlers_finished' : (gobject.SIGNAL_RUN_LAST, gobject.TYPE_NONE, ()), 'workers_update' : (gobject.SIGNAL_RUN_LAST, gobject.TYPE_NONE, - (gobject.TYPE_STRING, - gobject.TYPE_PYOBJECT)), + (gobject.TYPE_PYOBJECT,)), 'workers_finished' : (gobject.SIGNAL_RUN_LAST, gobject.TYPE_NONE, ()) @@ -57,6 +59,10 @@ class Listener(gobject.GObject): print("Ticker is finishing.") self.emit("ticker_finished", data[0]) return + elif data[0]=="ticker_update": + print("Ticker is running!") + self.emit("ticker_update") + return elif data[0]=="handler_update": print("Handlers are working.") self.emit("handlers_update") @@ -64,9 +70,10 @@ class Listener(gobject.GObject): elif data[0]=="work_update": print("Workers are working.") self.emit("workers_update", data[1]) + print("Signal: workers_update, ",data[1]) return else: - self.emit('update_ticker', data[0], data[1], data[2]) + return gobject.type_register(Listener) @@ -85,7 +92,9 @@ class Worker(): self.fractions.append(task) print(task) self.lock.acquire() + print("Lock acquired!") try: + print("Getting qsize") tasks = self.work_queue.qsize() self.fractions.append(tasks) print(tasks) @@ -100,7 +109,10 @@ class Worker(): #tasks = tasks/1000 time.sleep(0.001) self.work_queue.task_done() + self.lock.acquire() self.signal_queue.put("work_update", self.fractions) + print(self.fractions) + self.lock.release() self.work_queue.task_done() self.signal_queue.put("work_finished") @@ -121,6 +133,20 @@ class Handler(): self.ticker_queue.task_done() self.signal_queue.put("") +class Ticker(): + def __init__(self, sq, tq, i): + self.signal_queue = sq + self.ticker_queue = tq + self.tickers = i + self.iterations = 1 + + def go(self): + while self.iterations <= self.tickers: + self.ticker_queue.put(self.iterations) + time.sleep(0.01) + self.signal_queue.put("ticker_update", self.iterations) + self.iterations += 1 + self.signal_queue.put("ticker_finished") #def looprun(n): #for i in tickers: @@ -154,7 +180,7 @@ class Handler(): #for n in range(100): #looprun(n+1) -class ticker: +class t_gtk: global app @@ -173,7 +199,7 @@ class ticker: self.ticker_queue = multiprocessing.JoinableQueue() self.work_queue = multiprocessing.JoinableQueue() self.iterations = multiprocessing.Value('i', 0) - self.tickers = range(globvar.ticker_num) + self.tickers = globvar.ticker_num self.work_num = 0 self.processes = [] self.threads = [] @@ -192,7 +218,7 @@ class ticker: self.obj = builder.get_object self.obj("window").set_application(app) self.obj("window").set_wmclass("ticker","ticker") - self.obj("window").set_title("ticker - v.0.1.15") + self.obj("window").set_title("ticker - v.0.2.15") self.obj("window").show_all() self.obj("ticker_put_count_label").set_text(str(self.ticker_put_count)) self.obj("ticker_items_count_label").set_text(str(self.ticker_current_count)) @@ -207,27 +233,33 @@ class ticker: def run(self, argv): self.app.run(argv) + def update_ticker_put(self, index): + self.val = index*self.fraction + self.obj("ticker_put_count_label").set_text(str(index)) + self.obj("ticker_put_progress").set_fraction(self.val) + def update_ticker(self, obj, index1, index2, index3, data=None): self.val_1 = index1*self.fraction self.val_2 = index2*self.fraction self.val_3 = index3*self.fraction self.obj("ticker_put_count_label").set_text(index1) - self.obj("ticker_put_progressbar").set_fraction(self.val_1) + self.obj("ticker_put_progress").set_fraction(self.val_1) self.obj("ticker_items_count_label").set_text(index2) - self.obj("ticker_items_progressbar").set_fraction(self.val_2) + self.obj("ticker_items_progress").set_fraction(self.val_2) self.obj("ticker_processed_count_label").set_text(index3) - self.obj("ticker_processed_progressbar").set_fraction(self.val_3) + self.obj("ticker_processed_progress").set_fraction(self.val_3) - def update_work(self, obj, index): + def update_work(self, obj, index, data=None): + print(index) self.val_1 = index[0]*self.fraction self.val_2 = index[1]*self.fraction self.val_3 = index[2]*self.fraction self.obj("work_put_count_label").set_text(index[0]) - self.obj("work_put_progressbar").set_fraction(self.val_1) + self.obj("work_put_progress").set_fraction(self.val_1) self.obj("work_items_count_label").set_text(index[1]) - self.obj("work_items_progressbar").set_fraction(self.val_2) + self.obj("work_items_progress").set_fraction(self.val_2) self.obj("work_processed_count_label").set_text(index[2]) - self.obj("work_processed_progressbar").set_fraction(self.val_3) + self.obj("work_processed_progress").set_fraction(self.val_3) def ticker_Finished(self, obj, data=None): for t in range(20): @@ -237,11 +269,11 @@ class ticker: self.threads.append(t) self.obj("ticker_put_count_label").set_text(self.ticker_num) - self.obj("ticker_put_progressbar").set_fraction(1.0) + self.obj("ticker_put_progress").set_fraction(1.0) self.obj("ticker_items_count_label").set_text("0") - self.obj("ticker_items_progressbar").set_fraction(0.0) + self.obj("ticker_items_progress").set_fraction(0.0) self.obj("ticker_processed_count_label").set_text(self.ticker_num) - self.obj("ticker_processed_progressbar").set_fraction(1.0) + self.obj("ticker_processed_progress").set_fraction(1.0) def workers_Finished(self, obj, data=None): if self.process==None: @@ -251,11 +283,11 @@ class ticker: self.process = None self.obj("work_put_count_label").set_text(self.work_num) - self.obj("work_put_progressbar").set_fraction(1.0) + self.obj("work_put_progress").set_fraction(1.0) self.obj("work_items_count_label").set_text("0") - self.obj("work_items_progressbar").set_fraction(0.0) + self.obj("work_items_progress").set_fraction(0.0) self.obj("work_processed_count_label").set_text(self.work_num) - self.obj("work_processed_progressbar").set_fraction(1.0) + self.obj("work_processed_progress").set_fraction(1.0) def on_window_destroy(self,window): window.close() @@ -263,6 +295,7 @@ class ticker: def on_execute_clicked(self,widget): print("Creating Listener") listener = Listener(self.signal_queue) + listener.connect("ticker_update", self.update_ticker_put) listener.connect("handlers_update",self.update_ticker) listener.connect("handlers_finished",self.ticker_Finished) listener.connect("workers_update",self.update_work) @@ -273,8 +306,11 @@ class ticker: thread.start() print("Creating ticker queue") - for i in self.tickers: - self.ticker_queue.put(i) + t = Ticker(self.signal_queue, self.ticker_queue, self.tickers) + th = threading.Thread(target=t.go, args = ()) + th.deamon = True + th.start() + #th.join() print(self.ticker_queue.qsize()) for i in range(20): @@ -297,7 +333,7 @@ class ticker: -app = ticker() +app = t_gtk() app.run(sys.argv) diff --git a/parser/__pycache__/keywords.cpython-35.pyc b/parser/__pycache__/keywords.cpython-35.pyc new file mode 100644 index 0000000..fb86c43 Binary files /dev/null and b/parser/__pycache__/keywords.cpython-35.pyc differ diff --git a/parser/__pycache__/keywords.cpython-36.pyc b/parser/__pycache__/keywords.cpython-36.pyc new file mode 100644 index 0000000..377ed9b Binary files /dev/null and b/parser/__pycache__/keywords.cpython-36.pyc differ diff --git a/parser/parser.py b/parser/parser.py index c68f1c4..d2de8b8 100644 --- a/parser/parser.py +++ b/parser/parser.py @@ -2,20 +2,21 @@ # Python parser for c header files. # Used for creating corresponding NASM include files. -import keywords import os import sys import multiprocessing import threading +import time +import keywords keywords.init() num_cores = multiprocessing.cpu_count() work_queue = multiprocessing.JoinableQueue() signal_queue = multiprocessing.JoinableQueue() -test_folder = "..\\gtk" -print(test_folder) +test_folder = "/usr/include" filelist = [] +folderlist = [] processes = [] threads = [] @@ -30,8 +31,16 @@ class Listener: data = self.signal_queue.get() print(data) # Check if finished - if data[0]=="work_update": + if data[0] == "work_update": print("Workers are working.") + self.signal_queue.task_done() + return + elif data[0] == "work_finished": + print("Workers are finished.") + self.signal_queue.task_done() + return + else: + self.signal_queue.task_done() return class Worker: @@ -40,14 +49,17 @@ class Worker: self.work_queue = wq def go(self): + #while self.work_queue.get(2) != None: while True: - print("Working") + #print("Working") task = self.work_queue.get(2) - print(task) - time.sleep(0.001) + #print(task) + time.sleep(0.01) self.work_queue.task_done() + print(self.work_queue.qsize()) self.signal_queue.put("work_update") self.work_queue.task_done() + self.signal_queue.put("work_finished") class parser: def __init__(self, file): @@ -59,9 +71,9 @@ class parser: def get_token(self, keyword): token = "" if keyword in self.prep: - token = self.prep(keyword) + token = self.prep.values(keyword) if keyword in self.reg: - token = self.reg(keyword) + token = self.reg.values(keyword) return token def parse_preprocess(self, token): @@ -72,10 +84,10 @@ class parser: token = {} if word in self.prep: token.keys = word - token.values = self.prep(word) + token.values = self.prep.values(word) if word in self.reg: token.keys = word - token.values = self.reg(word) + token.values = self.reg.values(word) return token # Creates a list of elements per line in file, @@ -85,7 +97,7 @@ class parser: word = [w for w in line.split()] for w in word: token = self.tokenize_word(w) - if token.value == 'PREPROCESS': + if token.values == 'PREPROCESS': self.parse_preprocess(token) return tupline @@ -106,8 +118,9 @@ def sourcedir_filecnt(sourcedir): return cnt def sourcedir_foldercnt(sourcedir): - ### Return the number of folders, if it contains '*.h' files, in sourcedir - including subdirectories ### - global cnt + # Return the number of folders, if it contains '*.h' files, + # in 'sourcedir' - including subdirectories. + cnt = 0 global folderlist for folderName, subfolders, files in os.walk(sourcedir): if subfolders: @@ -127,7 +140,7 @@ def process_files(gui, source, dest): global destdir sourcedir = source destdir = dest - pool = mp.Pool(processes=num_cores) + pool = multiprocessing.Pool(processes=num_cores) pool.map(process_file, filelist) def process_file(data): @@ -175,11 +188,11 @@ def process_file(data): newfile = open(outputfile, "w") newfile.write(outfile) newfile.close() - -test_folder = get_script_path()+'\\..\\gtk' + + print(test_folder) print('Number of *.h files in directory: ',sourcedir_filecnt(test_folder)) -print(num_cores) +print('Number of processor cores: ',num_cores) print("Creating Listener") listener = Listener(signal_queue) @@ -193,7 +206,7 @@ print(threads) print("Creating work queue") for i in filelist: work_queue.put(i) - print(work_queue.qsize()) +print(work_queue.qsize()) for i in range(num_cores): w = Worker(signal_queue, work_queue) @@ -203,6 +216,21 @@ for i in range(num_cores): processes.append(p) print(processes) -work_queue.join() for i in range(num_cores): - work_queue.put(None) \ No newline at end of file + work_queue.put(None) + +print('Items on work queue: ',work_queue.qsize()) +work_queue.join() +print('Items on signal queue: ',signal_queue.qsize()) +signal_queue.join() + +print("Closing down workers") +for p in processes: + print("Closing down:",p) + p.join() +print(processes) + +print("Closing down Listener") +for t in threads: + t.join() +print(threads) \ No newline at end of file