From 35e8d57c9a062d9e56473773c01e0243707e5253 Mon Sep 17 00:00:00 2001 From: Jan Lerking Date: Thu, 15 Feb 2018 18:31:20 +0100 Subject: [PATCH 01/37] Updated for linux test --- multiprocessing_test.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/multiprocessing_test.py b/multiprocessing_test.py index 59f1e1a..2fb07fc 100644 --- a/multiprocessing_test.py +++ b/multiprocessing_test.py @@ -25,14 +25,12 @@ def sourcedir_filecnt(sourcedir): if file.lower().endswith('.h'): cnt += 1 filelist += [folderName+'/'+file] - print(folderName+'/'+file) - #print(filelist) return cnt def process_files(sourcedir, destdir): global filelist - outfile = '' for f in filelist: + outfile = '' inputfile = f encodings = ['utf-8', 'latin-1', 'windows-1250', 'windows-1252', 'ascii', 'big5', 'big5hkscs', 'cp037', 'cp273', 'cp424', 'cp437', 'cp500', @@ -62,7 +60,6 @@ def process_files(sourcedir, destdir): break print(os.path.basename(f)) for lines in fh: - print(lines) outfile = outfile+lines fh.close() outputfile = os.path.splitext(inputfile)[0]+'.inc' @@ -80,11 +77,14 @@ def process_files(sourcedir, destdir): def single_thread(): t1 = time.time() - sourcedir_filecnt(sourcedir) + cnt = sourcedir_filecnt(sourcedir) + print('Files in: '+str(cnt)) process_files(sourcedir, destdir) print('Single thread process time: '+str(time.time()-t1)) -sourcedir = 'C:/Users/dksojlg/Documents/gtk+-3.22.26' -destdir = 'C:/Users/dksojlg/Documents/include' +#sourcedir = 'C:/Users/dksojlg/Documents/gtk+-3.22.26' +sourcedir = '/usr/include' +#destdir = 'C:/Users/dksojlg/Documents/include' +destdir = '/data_2/include' single_thread() \ No newline at end of file From 812e365962b5542b3f93eb99181fffbf3bc295aa Mon Sep 17 00:00:00 2001 From: Jan Lerking Date: Thu, 15 Feb 2018 18:46:41 +0100 Subject: [PATCH 02/37] Added cpu_info --- multiprocessing_test.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/multiprocessing_test.py b/multiprocessing_test.py index 2fb07fc..f9f8c61 100644 --- a/multiprocessing_test.py +++ b/multiprocessing_test.py @@ -10,7 +10,9 @@ import sys import multiprocessing import time import io +import cpuinfo +cpu_info = cpuinfo.get_cpu_info() num_cores = multiprocessing.cpu_count() print(num_cores) @@ -87,4 +89,5 @@ sourcedir = '/usr/include' #destdir = 'C:/Users/dksojlg/Documents/include' destdir = '/data_2/include' -single_thread() \ No newline at end of file +print(cpu_info) +single_thread() #2543 files - Single thread process time: 32.53886866569519 \ No newline at end of file From a786528b854fc3bd5a0e3862b6232e7d58263e9b Mon Sep 17 00:00:00 2001 From: Jan Lerking Date: Thu, 15 Feb 2018 23:17:11 +0100 Subject: [PATCH 03/37] Added multi threading --- multiprocessing_test.py | 162 +++++++++++++++++++++++++++++----------- 1 file changed, 117 insertions(+), 45 deletions(-) diff --git a/multiprocessing_test.py b/multiprocessing_test.py index f9f8c61..452aa5d 100644 --- a/multiprocessing_test.py +++ b/multiprocessing_test.py @@ -11,12 +11,41 @@ import multiprocessing import time import io import cpuinfo +import queue +import threading +import time cpu_info = cpuinfo.get_cpu_info() num_cores = multiprocessing.cpu_count() print(num_cores) filelist = [] +exitFlag = 0 +queueLock = threading.Lock() +workQueue = queue.Queue(0) +threads = [] +class myThread (threading.Thread): + def __init__(self, threadID, name, q): + threading.Thread.__init__(self) + self.threadID = threadID + self.name = name + self.q = q + def run(self): + print ("Starting " + self.name) + process_data(self.name, self.q) + print ("Exiting " + self.name) + +def process_data(threadName, q): + while not exitFlag: + queueLock.acquire() + if not workQueue.empty(): + data = q.get() + process_file(data) + queueLock.release() + print ("%s processing %s" % (threadName, data)) + else: + queueLock.release() + time.sleep(1) def sourcedir_filecnt(sourcedir): ### Return the number of files, ending with '.h', in sourcedir - including subdirectories ### @@ -32,50 +61,53 @@ def sourcedir_filecnt(sourcedir): def process_files(sourcedir, destdir): global filelist for f in filelist: - outfile = '' - inputfile = f - encodings = ['utf-8', 'latin-1', 'windows-1250', 'windows-1252', 'ascii', - 'big5', 'big5hkscs', 'cp037', 'cp273', 'cp424', 'cp437', 'cp500', - 'cp720', 'cp737', 'cp775', 'cp850', 'cp852', 'cp855', 'cp856', - 'cp857', 'cp858', 'cp860', 'cp861', 'cp862', 'cp863', 'cp864', 'cp865', - 'cp866', 'cp869', 'cp874', 'cp875', 'cp932', 'cp949', 'cp950', 'cp1006', - 'cp1026', 'cp1125', 'cp1140', 'cp1250', 'cp1251', 'cp1252', 'cp1253', 'cp1254', - 'cp1255', 'cp1256', 'cp1257', 'cp1258', 'cp65001', 'euc-jp', 'euc-jis-2004', - 'euc-jisx0213', 'euc-kr', 'gb2312', 'gbk', 'gb18030', 'hz', 'iso2022-jp', - 'iso2022-jp-1', 'iso2022-jp-2', 'iso2022-jp-2004', 'iso2022-jp-3', - 'iso2022-jp-ext', 'iso2022-kr', 'iso8859-2', 'iso8859-3', 'iso8859-4', - 'iso8859-5', 'iso8859-6', 'iso8859-7', 'iso8859-8', 'iso8859-9', 'iso8859-10', - 'iso8859-11', 'iso8859-13', 'iso8859-14', 'iso8859-15', 'iso8859-16', 'johab', - 'koi8-r', 'koi8-t', 'koi8-u', 'kz1048', 'mac-cyrillic', 'mac-greek', - 'mac-iceland', 'mac-latin2', 'mac-roman', 'mac-turkish', 'ptcp154', - 'shift-jis', 'shift-jis-2004', 'shift-jisx0213', 'utf-32', 'utf-32-be', - 'utf-32-le', 'utf-16', 'utf-16-be', 'utf-16-le', 'utf-7', 'utf-8-sig'] - for e in encodings: - try: - fh = io.open(f, 'r', encoding=e) - fh.readlines() - fh.seek(0) - except UnicodeDecodeError: - print('got unicode error with %s , trying different encoding' % e) - else: - print('opening the file with encoding: %s ' % e) - break - print(os.path.basename(f)) - for lines in fh: - outfile = outfile+lines - fh.close() - outputfile = os.path.splitext(inputfile)[0]+'.inc' - outputfile = str(outputfile).replace(sourcedir, destdir) - print(outputfile) - if not os.path.exists(os.path.dirname(outputfile)): - try: - os.makedirs(os.path.dirname(outputfile)) - except OSError as exc: # Guard against race condition - if exc.errno != errno.EEXIST: - raise - newfile = open(outputfile, "w") - newfile.write(outfile) - newfile.close() + process_file(f) + +def process_file(data): + outfile = '' + inputfile = data + encodings = ['utf-8', 'latin-1', 'windows-1250', 'windows-1252', 'ascii', + 'big5', 'big5hkscs', 'cp037', 'cp273', 'cp424', 'cp437', 'cp500', + 'cp720', 'cp737', 'cp775', 'cp850', 'cp852', 'cp855', 'cp856', + 'cp857', 'cp858', 'cp860', 'cp861', 'cp862', 'cp863', 'cp864', 'cp865', + 'cp866', 'cp869', 'cp874', 'cp875', 'cp932', 'cp949', 'cp950', 'cp1006', + 'cp1026', 'cp1125', 'cp1140', 'cp1250', 'cp1251', 'cp1252', 'cp1253', 'cp1254', + 'cp1255', 'cp1256', 'cp1257', 'cp1258', 'cp65001', 'euc-jp', 'euc-jis-2004', + 'euc-jisx0213', 'euc-kr', 'gb2312', 'gbk', 'gb18030', 'hz', 'iso2022-jp', + 'iso2022-jp-1', 'iso2022-jp-2', 'iso2022-jp-2004', 'iso2022-jp-3', + 'iso2022-jp-ext', 'iso2022-kr', 'iso8859-2', 'iso8859-3', 'iso8859-4', + 'iso8859-5', 'iso8859-6', 'iso8859-7', 'iso8859-8', 'iso8859-9', 'iso8859-10', + 'iso8859-11', 'iso8859-13', 'iso8859-14', 'iso8859-15', 'iso8859-16', 'johab', + 'koi8-r', 'koi8-t', 'koi8-u', 'kz1048', 'mac-cyrillic', 'mac-greek', + 'mac-iceland', 'mac-latin2', 'mac-roman', 'mac-turkish', 'ptcp154', + 'shift-jis', 'shift-jis-2004', 'shift-jisx0213', 'utf-32', 'utf-32-be', + 'utf-32-le', 'utf-16', 'utf-16-be', 'utf-16-le', 'utf-7', 'utf-8-sig'] + for e in encodings: + try: + fh = io.open(data, 'r', encoding=e) + fh.readlines() + fh.seek(0) + except UnicodeDecodeError: + print('got unicode error with %s , trying different encoding' % e) + else: + print('opening the file with encoding: %s ' % e) + break + print(os.path.basename(data)) + for lines in fh: + outfile = outfile+lines + fh.close() + outputfile = os.path.splitext(inputfile)[0]+'.inc' + outputfile = str(outputfile).replace(sourcedir, destdir) + print(outputfile) + if not os.path.exists(os.path.dirname(outputfile)): + try: + os.makedirs(os.path.dirname(outputfile)) + except OSError as exc: # Guard against race condition + if exc.errno != errno.EEXIST: + raise + newfile = open(outputfile, "w") + newfile.write(outfile) + newfile.close() def single_thread(): t1 = time.time() @@ -84,10 +116,50 @@ def single_thread(): process_files(sourcedir, destdir) print('Single thread process time: '+str(time.time()-t1)) +def multi_thread(): + global filelist + global exitFlag + + t1 = time.time() + cnt = sourcedir_filecnt(sourcedir) + threadList = ["thread 1", "thread 2", "thread 3", "thread 4", "thread 5"] + nameList = filelist + threadID = 0 + + # Create new threads + for tName in threadList: + thread = myThread(threadID, tName, workQueue) + thread.start() + threads.append(thread) + threadID += 1 + + # Fill the queue + queueLock.acquire() + for word in nameList: + workQueue.put(word) + queueLock.release() + + # Wait for queue to empty + while not workQueue.empty(): + pass + + # Notify threads it's time to exit + exitFlag = 1 + + # Wait for all threads to complete + for t in threads: + t.join() + print ("Exiting Main Thread") + print('Files in: '+str(cnt)) + print('Multi thread process time: '+str(time.time()-t1)) + #sourcedir = 'C:/Users/dksojlg/Documents/gtk+-3.22.26' sourcedir = '/usr/include' #destdir = 'C:/Users/dksojlg/Documents/include' destdir = '/data_2/include' print(cpu_info) -single_thread() #2543 files - Single thread process time: 32.53886866569519 \ No newline at end of file +single_thread() #2543 files - Single thread process time: 1.3732633590698242 + +multi_thread() + From c212673b88658e3f48009a178cad00101ca68f9f Mon Sep 17 00:00:00 2001 From: Jan Lerking Date: Fri, 16 Feb 2018 21:02:48 +0100 Subject: [PATCH 04/37] Converted to executable linux script --- multiprocessing_test.py | 332 ++++++++++++++++++++-------------------- 1 file changed, 167 insertions(+), 165 deletions(-) mode change 100644 => 100755 multiprocessing_test.py diff --git a/multiprocessing_test.py b/multiprocessing_test.py old mode 100644 new mode 100755 index 452aa5d..bc5a4b8 --- a/multiprocessing_test.py +++ b/multiprocessing_test.py @@ -1,165 +1,167 @@ -# You are free to use and/or change this code for -# your own needs. - -# Original code (c)2018 Jan Lerking -# Program to test various scenarios of -# single thread, multi thread, pool and process - -import os -import sys -import multiprocessing -import time -import io -import cpuinfo -import queue -import threading -import time - -cpu_info = cpuinfo.get_cpu_info() -num_cores = multiprocessing.cpu_count() -print(num_cores) - -filelist = [] -exitFlag = 0 -queueLock = threading.Lock() -workQueue = queue.Queue(0) -threads = [] -class myThread (threading.Thread): - def __init__(self, threadID, name, q): - threading.Thread.__init__(self) - self.threadID = threadID - self.name = name - self.q = q - def run(self): - print ("Starting " + self.name) - process_data(self.name, self.q) - print ("Exiting " + self.name) - -def process_data(threadName, q): - while not exitFlag: - queueLock.acquire() - if not workQueue.empty(): - data = q.get() - process_file(data) - queueLock.release() - print ("%s processing %s" % (threadName, data)) - else: - queueLock.release() - time.sleep(1) - -def sourcedir_filecnt(sourcedir): - ### Return the number of files, ending with '.h', in sourcedir - including subdirectories ### - cnt = 0 - global filelist - for folderName, subfolders, files in os.walk(sourcedir): - for file in files: - if file.lower().endswith('.h'): - cnt += 1 - filelist += [folderName+'/'+file] - return cnt - -def process_files(sourcedir, destdir): - global filelist - for f in filelist: - process_file(f) - -def process_file(data): - outfile = '' - inputfile = data - encodings = ['utf-8', 'latin-1', 'windows-1250', 'windows-1252', 'ascii', - 'big5', 'big5hkscs', 'cp037', 'cp273', 'cp424', 'cp437', 'cp500', - 'cp720', 'cp737', 'cp775', 'cp850', 'cp852', 'cp855', 'cp856', - 'cp857', 'cp858', 'cp860', 'cp861', 'cp862', 'cp863', 'cp864', 'cp865', - 'cp866', 'cp869', 'cp874', 'cp875', 'cp932', 'cp949', 'cp950', 'cp1006', - 'cp1026', 'cp1125', 'cp1140', 'cp1250', 'cp1251', 'cp1252', 'cp1253', 'cp1254', - 'cp1255', 'cp1256', 'cp1257', 'cp1258', 'cp65001', 'euc-jp', 'euc-jis-2004', - 'euc-jisx0213', 'euc-kr', 'gb2312', 'gbk', 'gb18030', 'hz', 'iso2022-jp', - 'iso2022-jp-1', 'iso2022-jp-2', 'iso2022-jp-2004', 'iso2022-jp-3', - 'iso2022-jp-ext', 'iso2022-kr', 'iso8859-2', 'iso8859-3', 'iso8859-4', - 'iso8859-5', 'iso8859-6', 'iso8859-7', 'iso8859-8', 'iso8859-9', 'iso8859-10', - 'iso8859-11', 'iso8859-13', 'iso8859-14', 'iso8859-15', 'iso8859-16', 'johab', - 'koi8-r', 'koi8-t', 'koi8-u', 'kz1048', 'mac-cyrillic', 'mac-greek', - 'mac-iceland', 'mac-latin2', 'mac-roman', 'mac-turkish', 'ptcp154', - 'shift-jis', 'shift-jis-2004', 'shift-jisx0213', 'utf-32', 'utf-32-be', - 'utf-32-le', 'utf-16', 'utf-16-be', 'utf-16-le', 'utf-7', 'utf-8-sig'] - for e in encodings: - try: - fh = io.open(data, 'r', encoding=e) - fh.readlines() - fh.seek(0) - except UnicodeDecodeError: - print('got unicode error with %s , trying different encoding' % e) - else: - print('opening the file with encoding: %s ' % e) - break - print(os.path.basename(data)) - for lines in fh: - outfile = outfile+lines - fh.close() - outputfile = os.path.splitext(inputfile)[0]+'.inc' - outputfile = str(outputfile).replace(sourcedir, destdir) - print(outputfile) - if not os.path.exists(os.path.dirname(outputfile)): - try: - os.makedirs(os.path.dirname(outputfile)) - except OSError as exc: # Guard against race condition - if exc.errno != errno.EEXIST: - raise - newfile = open(outputfile, "w") - newfile.write(outfile) - newfile.close() - -def single_thread(): - t1 = time.time() - cnt = sourcedir_filecnt(sourcedir) - print('Files in: '+str(cnt)) - process_files(sourcedir, destdir) - print('Single thread process time: '+str(time.time()-t1)) - -def multi_thread(): - global filelist - global exitFlag - - t1 = time.time() - cnt = sourcedir_filecnt(sourcedir) - threadList = ["thread 1", "thread 2", "thread 3", "thread 4", "thread 5"] - nameList = filelist - threadID = 0 - - # Create new threads - for tName in threadList: - thread = myThread(threadID, tName, workQueue) - thread.start() - threads.append(thread) - threadID += 1 - - # Fill the queue - queueLock.acquire() - for word in nameList: - workQueue.put(word) - queueLock.release() - - # Wait for queue to empty - while not workQueue.empty(): - pass - - # Notify threads it's time to exit - exitFlag = 1 - - # Wait for all threads to complete - for t in threads: - t.join() - print ("Exiting Main Thread") - print('Files in: '+str(cnt)) - print('Multi thread process time: '+str(time.time()-t1)) - -#sourcedir = 'C:/Users/dksojlg/Documents/gtk+-3.22.26' -sourcedir = '/usr/include' -#destdir = 'C:/Users/dksojlg/Documents/include' -destdir = '/data_2/include' - -print(cpu_info) -single_thread() #2543 files - Single thread process time: 1.3732633590698242 - -multi_thread() - +#!/usr/bin/env python3.5 + +# You are free to use and/or change this code for +# your own needs. + +# Original code (c)2018 Jan Lerking +# Program to test various scenarios of +# single thread, multi thread, pool and process + +import os +import sys +import multiprocessing +import time +import io +import cpuinfo +import queue +import threading +import time + +cpu_info = cpuinfo.get_cpu_info() +num_cores = multiprocessing.cpu_count() +print(num_cores) + +filelist = [] +exitFlag = 0 +queueLock = threading.Lock() +workQueue = queue.Queue(0) +threads = [] +class myThread (threading.Thread): + def __init__(self, threadID, name, q): + threading.Thread.__init__(self) + self.threadID = threadID + self.name = name + self.q = q + def run(self): + print ("Starting " + self.name) + process_data(self.name, self.q) + print ("Exiting " + self.name) + +def process_data(threadName, q): + while not exitFlag: + queueLock.acquire() + if not workQueue.empty(): + data = q.get() + process_file(data) + queueLock.release() + print ("%s processing %s" % (threadName, data)) + else: + queueLock.release() + time.sleep(1) + +def sourcedir_filecnt(sourcedir): + ### Return the number of files, ending with '.h', in sourcedir - including subdirectories ### + cnt = 0 + global filelist + for folderName, subfolders, files in os.walk(sourcedir): + for file in files: + if file.lower().endswith('.h'): + cnt += 1 + filelist += [folderName+'/'+file] + return cnt + +def process_files(sourcedir, destdir): + global filelist + for f in filelist: + process_file(f) + +def process_file(data): + outfile = '' + inputfile = data + encodings = ['utf-8', 'latin-1', 'windows-1250', 'windows-1252', 'ascii', + 'big5', 'big5hkscs', 'cp037', 'cp273', 'cp424', 'cp437', 'cp500', + 'cp720', 'cp737', 'cp775', 'cp850', 'cp852', 'cp855', 'cp856', + 'cp857', 'cp858', 'cp860', 'cp861', 'cp862', 'cp863', 'cp864', 'cp865', + 'cp866', 'cp869', 'cp874', 'cp875', 'cp932', 'cp949', 'cp950', 'cp1006', + 'cp1026', 'cp1125', 'cp1140', 'cp1250', 'cp1251', 'cp1252', 'cp1253', 'cp1254', + 'cp1255', 'cp1256', 'cp1257', 'cp1258', 'cp65001', 'euc-jp', 'euc-jis-2004', + 'euc-jisx0213', 'euc-kr', 'gb2312', 'gbk', 'gb18030', 'hz', 'iso2022-jp', + 'iso2022-jp-1', 'iso2022-jp-2', 'iso2022-jp-2004', 'iso2022-jp-3', + 'iso2022-jp-ext', 'iso2022-kr', 'iso8859-2', 'iso8859-3', 'iso8859-4', + 'iso8859-5', 'iso8859-6', 'iso8859-7', 'iso8859-8', 'iso8859-9', 'iso8859-10', + 'iso8859-11', 'iso8859-13', 'iso8859-14', 'iso8859-15', 'iso8859-16', 'johab', + 'koi8-r', 'koi8-t', 'koi8-u', 'kz1048', 'mac-cyrillic', 'mac-greek', + 'mac-iceland', 'mac-latin2', 'mac-roman', 'mac-turkish', 'ptcp154', + 'shift-jis', 'shift-jis-2004', 'shift-jisx0213', 'utf-32', 'utf-32-be', + 'utf-32-le', 'utf-16', 'utf-16-be', 'utf-16-le', 'utf-7', 'utf-8-sig'] + for e in encodings: + try: + fh = io.open(data, 'r', encoding=e) + fh.readlines() + fh.seek(0) + except UnicodeDecodeError: + print('got unicode error with %s , trying different encoding' % e) + else: + print('opening the file with encoding: %s ' % e) + break + print(os.path.basename(data)) + for lines in fh: + outfile = outfile+lines + fh.close() + outputfile = os.path.splitext(inputfile)[0]+'.inc' + outputfile = str(outputfile).replace(sourcedir, destdir) + print(outputfile) + if not os.path.exists(os.path.dirname(outputfile)): + try: + os.makedirs(os.path.dirname(outputfile)) + except OSError as exc: # Guard against race condition + if exc.errno != errno.EEXIST: + raise + newfile = open(outputfile, "w") + newfile.write(outfile) + newfile.close() + +def single_thread(): + t1 = time.time() + cnt = sourcedir_filecnt(sourcedir) + print('Files in: '+str(cnt)) + process_files(sourcedir, destdir) + print('Single thread process time: '+str(time.time()-t1)) + +def multi_thread(): + global filelist + global exitFlag + + t1 = time.time() + cnt = sourcedir_filecnt(sourcedir) + threadList = ["thread 1", "thread 2", "thread 3", "thread 4", "thread 5"] + nameList = filelist + threadID = 0 + + # Create new threads + for tName in threadList: + thread = myThread(threadID, tName, workQueue) + thread.start() + threads.append(thread) + threadID += 1 + + # Fill the queue + queueLock.acquire() + for word in nameList: + workQueue.put(word) + queueLock.release() + + # Wait for queue to empty + while not workQueue.empty(): + pass + + # Notify threads it's time to exit + exitFlag = 1 + + # Wait for all threads to complete + for t in threads: + t.join() + print ("Exiting Main Thread") + print('Files in: '+str(cnt)) + print('Multi thread process time: '+str(time.time()-t1)) + +#sourcedir = 'C:/Users/dksojlg/Documents/gtk+-3.22.26' +sourcedir = '/usr/include' +#destdir = 'C:/Users/dksojlg/Documents/include' +destdir = '/data_2/include' + +print(cpu_info) +single_thread() #2543 files - Single thread process time: 1.3732633590698242 + +multi_thread() + From 77e4934ba5b3f7e86e617d91e7748623ad29dafc Mon Sep 17 00:00:00 2001 From: Jan Lerking Date: Fri, 16 Feb 2018 21:15:19 +0100 Subject: [PATCH 05/37] Renamed 'multi_thread()' -> 'syncronized_thread()' --- multiprocessing_test.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/multiprocessing_test.py b/multiprocessing_test.py index bc5a4b8..9b73d04 100755 --- a/multiprocessing_test.py +++ b/multiprocessing_test.py @@ -118,7 +118,7 @@ def single_thread(): process_files(sourcedir, destdir) print('Single thread process time: '+str(time.time()-t1)) -def multi_thread(): +def syncronized_thread(): global filelist global exitFlag @@ -161,7 +161,7 @@ sourcedir = '/usr/include' destdir = '/data_2/include' print(cpu_info) -single_thread() #2543 files - Single thread process time: 1.3732633590698242 +single_thread() #2543 files - Single thread process time: 1.3732633590698242 sec. -multi_thread() +syncronized_thread() #2543 files - Syncronized 5 threads process time: 80.25179100036621 sec. From 9ab6f60f1474580b448b1447be0815962b638cad Mon Sep 17 00:00:00 2001 From: Jan Lerking Date: Fri, 16 Feb 2018 23:05:05 +0100 Subject: [PATCH 06/37] Added pool.map testing --- multiprocessing_test.py | 65 ++++++++++++++++++++++++++++++++++------- 1 file changed, 54 insertions(+), 11 deletions(-) diff --git a/multiprocessing_test.py b/multiprocessing_test.py index 9b73d04..931843d 100755 --- a/multiprocessing_test.py +++ b/multiprocessing_test.py @@ -9,16 +9,19 @@ import os import sys -import multiprocessing +import multiprocessing as mp import time import io import cpuinfo import queue import threading import time +import random +import string +import shutil cpu_info = cpuinfo.get_cpu_info() -num_cores = multiprocessing.cpu_count() +num_cores = mp.cpu_count() print(num_cores) filelist = [] @@ -92,15 +95,15 @@ def process_file(data): except UnicodeDecodeError: print('got unicode error with %s , trying different encoding' % e) else: - print('opening the file with encoding: %s ' % e) + #print('opening the file with encoding: %s ' % e) break - print(os.path.basename(data)) + #print(os.path.basename(data)) for lines in fh: outfile = outfile+lines fh.close() outputfile = os.path.splitext(inputfile)[0]+'.inc' outputfile = str(outputfile).replace(sourcedir, destdir) - print(outputfile) + #print(outputfile) if not os.path.exists(os.path.dirname(outputfile)): try: os.makedirs(os.path.dirname(outputfile)) @@ -118,7 +121,7 @@ def single_thread(): process_files(sourcedir, destdir) print('Single thread process time: '+str(time.time()-t1)) -def syncronized_thread(): +def multi_thread(): global filelist global exitFlag @@ -155,13 +158,53 @@ def syncronized_thread(): print('Files in: '+str(cnt)) print('Multi thread process time: '+str(time.time()-t1)) +def async_process(num): + t1 = time.time() + cnt = sourcedir_filecnt(sourcedir) + + pool = mp.Pool(processes=num) + #[pool.apply_async(process_file, args=(f,)) for f in filelist] + pool.map(process_file, filelist) + #output = [p.get() for p in results] + #print(output) + + # Setup a list of processes that we want to run + #processes = [mp.Process(target=rand_string, args=(5, output)) for x in range(8)] + #processes = [mp.Process(target=process_files, args=(sourcedir, destdir)) for x in range(num)] + + # Run processes + #for p in processes: + #p.start() + + # Exit the completed processes + #for p in processes: + #p.join() + + # Get process results from the output queue + #results = [output.get() for p in processes] + + #print(results) + print('Files in: '+str(cnt)) + print('Multi ('+str(num)+') processes time: '+str(time.time()-t1)) + +def cleanup(dest): + shutil.rmtree(dest) + #sourcedir = 'C:/Users/dksojlg/Documents/gtk+-3.22.26' sourcedir = '/usr/include' #destdir = 'C:/Users/dksojlg/Documents/include' destdir = '/data_2/include' -print(cpu_info) -single_thread() #2543 files - Single thread process time: 1.3732633590698242 sec. - -syncronized_thread() #2543 files - Syncronized 5 threads process time: 80.25179100036621 sec. - +#print(cpu_info) +#single_thread() #2543 files - Single thread process time: 1.3732633590698242 sec. +#os.rmdir(destdir) +#multi_thread() #2543 files - Syncronized 5 threads process time: 80.25179100036621 sec. +#os.rmdir(destdir) +async_process(16) #2543 files - 2 processes, process time: 1.2379400730133057 sec. +#cleanup(destdir) +#async_process(4) #2543 files - 2 processes, process time: 2.6622860431671143 sec. +#cleanup(destdir) +#async_process(6) #2543 files - 2 processes, process time: 2.6622860431671143 sec. +#cleanup(destdir) +#async_process(8) #2543 files - 2 processes, process time: 2.6622860431671143 sec. +#cleanup(destdir) From 7685b8c6a833451aa70507185d0b9c350e19f65a Mon Sep 17 00:00:00 2001 From: Jan Lerking Date: Fri, 16 Feb 2018 23:43:30 +0100 Subject: [PATCH 07/37] Updated h2inc to use multiprocessing --- __pycache__/h2inc.cpython-35.pyc | Bin 1770 -> 3748 bytes __pycache__/h2inc_gui.cpython-35.pyc | Bin 6608 -> 6817 bytes async_multiprocessing_test.py | 34 +++++ h2inc | 1 + h2inc.py | 199 +++++++++++++++------------ h2inc_gui.py | 3 +- multiprocessing | 1 + pool_test.py | 20 +++ 8 files changed, 170 insertions(+), 88 deletions(-) create mode 100644 async_multiprocessing_test.py create mode 120000 h2inc mode change 100644 => 100755 h2inc.py create mode 120000 multiprocessing create mode 100644 pool_test.py diff --git a/__pycache__/h2inc.cpython-35.pyc b/__pycache__/h2inc.cpython-35.pyc index 57272df03dbbc14584bd5642e111b01b498e81e5..1df678b448d8ede54381f4d4cfdc83334602eb8a 100644 GIT binary patch literal 3748 zcmZt}+jiT=aR8FwMU-SomMpt*ET>IUHm1yrcv0#)ZezDin<{SQl(HJtfiNHiUI1zr zJd$waKGZp<&;5e-wGaJ+e#yS}DZh}X&g@bm<5QB@ot>STo!btE8;gsD&f(t<@dm(u zVeVrS{DdC)9~zK;77PInuVD)gtt-3@hp8(#4TotB+c3<)$bmx#Mp-z_f;E+rgToy3 zNLC)YG!__k7xY+UPInP71@4j_mjOv!2EHN68-NaAR>LcRIl#PzR{;xv3mRSnBq56$ zzR61OmzKUI3t+EgtNhp%(jNw)uNE|@2UOr$OhO-ro(w&o(j)0imhjqw^D~G6`W9RQ zys_ZvQ5U{S!D|~{)0l?y7jT&-?C0=01D6@-+wg|S$tMI@ARfRa`6>l15*4T6(q^M> z5#g<#1uZuXF|o~%#A2O3fX?X#bck`kl-U!kU}A5de9ORh*29vc>`1EgnL8Y)%rH9h zh06IM%CjIoQMqm~Bs*oSGGh^hF;VAbG!dSUflw(gj8&l<4bc~S?#NdKIXN~3w|B*)??m28nGK2e~7^3L9_h*e&*H_dR9pzjKgI7VJH8b98q zwd8~IkbF4jO&uii=k}vvpUf%HAgp|CDlYcv<^m6YE1*aZHhrjHIR`;Z*y z(rAH8*J+cZrsHRGU?`Y6erwPnmy!?nN@=a5%Azl0%2}~MB2;lKBF~pnCs`?_9AmCZ zGnKO5)>tjh)5{i#knAQ%Ia+sdpWy4LbFAoelZyW!JIvK5y(#r5rzm^q)>0`lMaYyN zQ+Q0tL8@3(uBb{;aiW?;rHCrfI^YK2Ex=8{+kkffe*pLn;9bCX0q+5B0p16E5AcV8 zKLUIo@B_dP0e=iw0{jW!M}Q9i%YfT}4*@HHRlpix9dHM*0oVj=0k#2m0Y3)(Dc~c( zp8-Aw`~>g`;Libn0r)B4Q^03{zXbdh;I9Ec1N;r(Zvj6C{2kyP;1__;0e=tp2f%&6 z7l2;^{t>VPcmVhk@DT7%fL{R~0sa~A72wx^F5oes2Z(?^^pMNqxv#I_(GlO#i~8(V zV;8WeudHp&mrIKaC;9k9=Rzmkf;832K>@@+E&5Q2#DLj0I8|PrKC=#<)6n5skTJQW z@8P8bx9FOl{@d!XK!>lY6=$KJgMNORX>`VI>=RuU;sRV|`NaML#6^e~;SCiU!W)jI zD`0s_MwD`5U)5MflE4m72;LsufZS%;@JKxK<6bT z9Q}yqFF7~}EjWm&!ejvlS`E@S5B-~bYuIRB7<=?B8SaTRbz2`)(y=-D5rN^t5<6()jJ-jZ11?*uGA~a zK1q^7AfsBPR@0JZFh_81RZEuVSNfniL_JxWTlwelOzc1;dD?5Y8@uJ&46M(d=ir?=xG@J)jQi1vdz?6fC}@|(bdIOZfI!Yi5LX(L}kY@B};=0 zmnT6tF82eOEM0>pjkB`OxBSrXCh2Wz z!IX7YT32=usdO~JU{v2*+38l}S=rzOuU~)X!=Q3PgNGl=9C#l@0wf z=6<^~xl77@LXUh-1JJwOv6iXXT(>s$m!TqIR`qw8piSyrQZyE<6@u1M8Jjzqf_2B* zwBMzjyP9%?C>Jfk(migw`NRDU_cpq(;cm$MFQ8t58ihAtO*=&0*3j#VUMBpvXTN0c z(Z3JVz4)?}QTAx8($Ax4s8S<$tk2Lr4t0yEa5AEh(B4~iszp~`gx;&EoDrQa`3(T1 zt+=4Mv~J>=k{41fk0!%7n9_2xk)|LHXi!DnzQ5A{euQ9?1ly$Y2H%?mDjQEI8|aTx zc1(FdifiozxAnTAoMnY>=ymg_`zd1zYcr?4?t8iARk=r_2q#1TF}GXtJsOG>8~$!s SMVljc{k4mW#g$^NxcxtC4uNn0 literal 1770 zcmZuxO>f&q5S`_xNK5{xQ!5S7Iyt380z1vc7sCxu9}A;^3m6b;xGcG{rBWibyMh!$ zLJPR}qCca5X|Fw{#~yR)o0aW4L22)3b~rPfeQ#!WI-S&?{L=e9A^M9RJRbBnSo&9p z690lKqTanOs3)A=qspU^Pd%ST0rdhBx85Q3Lb^muM17(>W<75?nQ&yDQt49}&~NZr zqcWs2a(bJ}n97#ZJN)6(EP012=p^%O(&S3ZYA}{|&8be5g758PX!8p+`4xM|n^LsXi@mTAvP($^^9V zxb7|-yO)9v@_-D&{xc#(OkUF2WQWcWc$$SLiVM8<_O9Ns@?Pv`k@aeAgIQkvY{ROa zNo8AdT!~VaIZ)Ow#>S@ox+h)k;tmIzv)^6P=zu-9J-$tC?gdi{Sc11^g zDpIT`o_Yk^2Y3fCIK1aj=97<*{?Q=~797|>Nxh-@V}|Sae3QX99)o{}8TbO|U~b{l zF^nrh)Fv6Q;ZryF9Rq@Hk)LeR8D5=c!O4A5+`L@WDVBZ#fz|`@=g+tmK*$hH;JTr7 z-Jl{i{^8LiLLR{^Wu9swPGKwvUFYq+Qd_)dG*^Q0d7mCKM@2nqj=scR^M@0-+ zkI#2#+2T$yNZ4|&C}E<(cud0<4HF(#WI$D);R5L{6EZ0+61EN3oDQDboD&v)1^eBW z3xz-G#(fmIy5W%0u*08(zu?WPpvg9!p;=C|_{5T%Wh<1U?>gASBv~=b)>R6;Y*06H zY~x1BM%4wVm^PJE466-`s@78Z&>iPCXmTEQ-Bbf3=f?WlC>tZP$_r`zONsfPDZI

+EVwE)Zk9@pJ9?O|a z@?l%s4p%OVB3{KC*emt~4BR$2%3E#Q?wYx9 zceHXCxWn6IWxt#bY|QweMzxD5hoE}Q8aKVWYFt~o)DQaE70e@OTZf~%oL2IP=T+}O Tv_)I&h_=VYx<9YoX>a`t%Wigi diff --git a/__pycache__/h2inc_gui.cpython-35.pyc b/__pycache__/h2inc_gui.cpython-35.pyc index 2f74536f662831cefa526cdba0b0b68f5e8c7a8c..38443b4e657532bdce36945974900011e17b5c15 100644 GIT binary patch delta 2411 zcmb7_&u<$=6vyAJf2{4S6Fatd9mh%R+Bn(#Zhj&_0#Qnd1Q8GrsZ$Y($A!8miPjql zMQMWwP{g4EjdJ0F;18&TA~;1zy>X{*tmW8=_(K~#aQ$!y@Fl=z z063#gP<3?R8qJ`ek2+!Ce(FSk2dL8{+ZuzK0XztY4452iG}dXn?@^;<7=>Cd@KDR} zO`PZhj$9;$ahfqV2I#_j07c_iN{f?k9BTbC0F=oQf-QQ+fXBg2fSUx%P2ppTcO&K; z+IC~DCPb1`ikzm&G?K+F-8xtTo&=u2lmwVNpd3uVWv|JMu0j};F@45hM&2E zih-M2z|(LW;2J6)0G@#_0FjyuFiU3HgHGlkF-_nY_Hw(j_E54))(|m<6aoc#Ng`Pi z@pKzP9uk@k10Mo+1mnfL55N^U9S80OtSNGi$hl7P*#S!c8g_7-IS`)*a@jng@|~z= zoZ?;}W#LkX^Eq;ei@yVfLV@}P<&18vGEk`!??sZ&p+!4GYVuJ*7BC`cE3lG^ut?-0 zl0`%xGCIoXmyr1iu5^`?gyQT|r)t+*kVQ78=p~|;o&*NWvm6 zbvVb!p-8uPLWMt0ljDjqDPp>HD#e#@{p=)+tAHX3I6=S(1)S=ZwSh|=vL>N6)fB;} zfzJS+1%4iJbor)fXq92KLDFK=kUjo zeWxIsqL__7vd3gMhfO1RmGkRc6|y+Xdb#&&>%7yW>>cu)ltV-O*}KFV$FaU6X3;Lo za=Lxnf;fo{GvoMSEU&SW__zNkyC7{$**uSp9rX0&SMhl$A2Z}89FCY zYZ=;w6m(gQiceB6_JxoV$L9h*^#Bw1Q-daczVOuya5D(|e)*-jp@H*2{F7Q?55xw()4D9Pr3xOMw_w9xcoO|#oz&n>Y0RE%REnOU$@Z!bwm9@8=B^ir_jUDoWYbD!}c`FC_HYU8`=heE~2-dfEol3RgtA6aALEBMH zp!FSm>ciLoGsGS1$`Yku9>+Fo_$phjXWz2N8a|XKnJ`StKHvuz-!S+=lVNGAd%YKS9jChG<|Oc$oUM7$ETTQ`tyjF& z?*03l^U|rg`w!?**Pw_$Y;W+xIgI|&+bh$j4Oi{3cr`u#@(5Zwqq31HaMop0md)c+ yZw2h6aUqFM-N43R#$_XE7)IRiiJ#Itcp*vyysf4@Kk@CrRQQ`Piv(iKVE+N2D!9-9 delta 2412 zcmb7F&2Jk;6o0e!uGh{cwv*UvJ5J)*PV9B!B;>0XqzF(!sUSiMl-3}nDC3Gc&{XR^ zAe3SWON9#vY3P9q5(kbPP*e&20(#8ZhhDfLA%r*}BqVtA*5j{6K%&g^%)Ix0@Auxk znf-eEd^4OJACG){y;WQOF8rp(UKVw2zj)wS9rK-yJ9m#>dVTLm1OHF{MsWmiG=C63 zX{8ot@Cm?S5V{jxO$^`~sY1X5G%L3IYjDfGH~}d5O55XrXQv_$+vHSUNn6kpwk`G*3wLLrC%%FGx@}1R4&(4yHUe zkh1tjk7jd(og2pPCMwSLgpH~pv;}gAiNEwhK2P(!az?jauPBJ;jKs5OFbE+z2B>KZ zm>~=_;gZU@K)?kA7U|Gz>)%=~L@kG=BP}PR;BN7$hW8b+=gTVGB86L2d0=Tv!28*V z)sT7Tzd4})2nWw^j78ZPGOWC$bjg-zxugR2E*YL^LfkSF8>kvA*(i{_)C5!_ppq=q z-$7f{&6tggAy@zS3!-S|vSl(atGIZf{R2~Rw^23Z93_VWb@s9f>d>a6P*%iz$o?Po zL5GX$n5kHR|vb?xvJ+brn!E%^ez#mu6-J%qWVt4$BR?=Dd z5!ihL*asOnHK1XFXF?+k#{qC3!AAgZ+~ffGk2W6~;!(0PsY%eb1`Kx(oZ!A&5x*tt zH)_V!t2g!zUUPSlf@`C2LVgK)r{Pv{x2E$GJdsY2*PC7^P=DV)oPjf3L95SjG_xo` zd~d&%r4nr6%3+*lT~Jd2G~69A~QGPs1#t$C$ Date: Mon, 19 Feb 2018 04:36:37 +0100 Subject: [PATCH 08/37] Added multiprocessing testing --- __pycache__/h2inc.cpython-35.pyc | Bin 3748 -> 3818 bytes __pycache__/h2inc_gui.cpython-35.pyc | Bin 6817 -> 6564 bytes h2inc_gui.py | 16 ++--- mp_counter.py | 29 ++++++++ mp_test.py | 40 +++++++++++ multiprocessing_test.py | 104 +++++++++++++++++++++++++++ 6 files changed, 181 insertions(+), 8 deletions(-) create mode 100644 mp_counter.py create mode 100644 mp_test.py diff --git a/__pycache__/h2inc.cpython-35.pyc b/__pycache__/h2inc.cpython-35.pyc index 1df678b448d8ede54381f4d4cfdc83334602eb8a..2dec8fc417e5ee7a12f514df8a54c322b679b88d 100644 GIT binary patch delta 467 zcmYk1ze@u#6vtog(!16sQngi)wrZ_P{Z$bJM<<;G(MiFU(wtCfdq?g%h)M?qag?mO ziM#kuh#>9(CpS0$0rBNr4CM2^`6lnZyobV9VP=%`fmZGM@IC|Z3ICRbdJ}i}ay!%C z_G~!Gz$L)p5C{q)0avJpK{z1XSdV~6f=I#?5aAJc%L9r>hu;3h>1Vj!*uB!3zy`!lcjC^ zj+{k9-Sc9$iz1{5FttxHR>ssj&pUC2BlXR9W((-*R5R$|6uoXFr|>Lu2IB^E1|_vW UxJb&XKbTJwW--o&Sdy*$0@s3LWB>pF delta 477 zcmYk0!7D^T9LImZdGGC8%&=>%D31_&6elE#gWO0tU`yJ}TW#&zT{CZskOSqcNiL3Z zaFeTl!ga1r{1s;2O6J#mf1jD}cc$NW@Dz;qGuF9Vx-Ib>z$g4$#PFKg^8If7YMHyh zJ-7iV<$z~^`zhyvXMuO5+zY%Dcvs2=;1oDZx$pJs9C7e1B6End>fU+ANwwn6U(J}$ zF96C$5y}Xs1CS6{Kghu)fy99e0@nz5Qh;3!uoL1|=n7^YX`e?j;Z-5J?1?r-<0z8y zuu`o>p`$ah(Go|Ic5O;yMHEb`Eqyl6+IcnN`GrMuA+tun$zTQx*xFktj2|}x~xXcs2ZY;HuWFoq}qE35t zsF)M|oPEj;VE)f+Ssr3%bKeR>7^TUe;RlGrFu?(KJU@wX5ArV?qh#2Rz!lvvEVm-! zw#mVe8C(G_iK~~vtAQB=tpsjcD%lhdEYl5nW&z2H=Z3n?6OLz?JZqSt36ACJq1j4P zY|7D>ET`^hjTj7|0eTU%80;ZV2ISk@M`tD%Rd6HAe) zLaHpc>Qz14FwreII(Rl$sRN*D3gc|7t$#1?tbt{eqv}^Yw`$euV1hy?B&(o17#Fs^ z!t(XB&(4a2Y<=FeK{P45QJ8r=JTJWdda3YAq7P?C@YcpqPq<&jPB ziCt&Ig;Ku|QNX>XM%Vp2@IaHpS)`h^Dw`kTe%9MG0qXO#%V2661^oxK~ zQ9x>ajv{{~m96#EMy{bB1_?+t6bU?ndjy*YucuhO`*57fK6o6;^v)=Vj9GPGzG)-)N3Qh<~>o|b~+LRUqDM(dYp=b`6jdzo5Snpc1 zYntR@i6FJ-ju2e?bKnr6B1@c5FI>1*szB_4Q!hSoiBzd4-kXh;NG-DV^PBfG^WMBS zzg2#&YWYki`GWnx9zW4uMoxdYDD4h}Wv4?b4{1v-M&x%6)4|=ZiGCmolNug*@ zt8rpfh|y2BNrd_Z@*^SZW5h^FAw{++1x;;IE+@%)ju;tYOp6h%L?^|9Hqk#NO;bHn zRs60_BovHVpAt{BQ9397)Sgo&?kC5pELnH$R>Sd{y3>skLYU(DR{i!Si#2s)qZ>yR zikn;Nw{DBq&AB2IyUC*vUr5zxNqnDrQz(H~newCX$5H0T5XXg`o(I22>0jEXP*DT~ zif$N|QtK9UwZVZ-P1$BzcEz|K?at(wn*YR~bh&5cpWGTb+ zP2R8^-8Z`l^i4GM4Xe>=blWP%Yk+cukH8-v3gr(`M#X#eAM+IY6cMtD(0O>Q}U z5M4EWK8L1xL=JHtaRIS__yRGF$Rj=jl$9{gA}Y`nBz&{cgngM5peUw=7YX+&D@h-i zveC}lT!pqVxpJ0c+xZp5Rm37fCVLH~>xfSg`w3zSVQ!zJbP<7B_O4I%Tn_dAWS<24 zUHH5K0HIl`bsi0TMQK^=9xnIGQ{;J 0: + tempstr = 'Number of headers: '+str(filecnt) + temptot = 'Total progress: 0 of '+str(filecnt) + print ('Source directory: ', sourcedir.get()) + self.destlabel.config(state=NORMAL) + self.destentry.config(state=NORMAL) + self.destdir_button.config(state=NORMAL) + self.infofiles.config(text=tempstr) + self.totallabel.config(text=temptot) + self.filecnt = filecnt + foldercnt = sourcedir_foldercnt(root.directory) + if foldercnt > 0: + tempstr = 'Number of folders: '+str(foldercnt) + self.infofolders.config(text=tempstr) + + def select_destdir(self, destdir): + root.directory = filedialog.askdirectory() + if root.directory: + destdir.set(root.directory) + print ('Destination directory: ', destdir.get()) + self.incchkbox.config(state=NORMAL) + self.infofolders.config(state=NORMAL) + self.infofiles.config(state=NORMAL) + self.translate_button.config(state=NORMAL) + self.cfilelabel.config(state=NORMAL) + self.totallabel.config(state=NORMAL) + + def translate(self, destdir, sourcedir, addinc): + doinc = addinc.get() + dest = destdir.get() + source = sourcedir.get() + if doinc == 'yes': + dest = dest+'/include' + print(os.path.exists(os.path.dirname(dest))) + if not os.path.exists(os.path.dirname(dest)): + try: + os.makedirs(os.path.dirname(dest)) + except OSError as exc: # Guard against race condition + if exc.errno != errno.EEXIST: + raise + destdir.set(dest) + print ('Destination directory: ', destdir.get()) + process_files(source, dest) + + def cfileprogress_update(cnt): + self.cfilevar = cnt + + def currentfile_update(current): + self.currentfile = 'Current file: '+current + +root = Tk() +root.update() +#root.minsize(350, 210) +#width = (root.winfo_screenwidth()/2)-(350/2) +#height = (root.winfo_screenheight()/2)-(210/2) +#root.geometry('+%d+%d' % (width, height)) +root.resizable(False, False) +f_progress = file_progress(root) +root.mainloop() cpu_info = cpuinfo.get_cpu_info() num_cores = mp.cpu_count() From 81fff6e01c1fc89b321432dce063f577c334cb55 Mon Sep 17 00:00:00 2001 From: Lerking <33354709+Lerking@users.noreply.github.com> Date: Tue, 20 Feb 2018 06:30:18 +0100 Subject: [PATCH 09/37] Add files via upload --- .../async_multiprocessing_test.py | 34 ++ Multiprocessing test/mp_counter.py | 29 ++ Multiprocessing test/mp_test.py | 40 +++ Multiprocessing test/multiprocessing | 1 + Multiprocessing test/multiprocessing_test.py | 314 ++++++++++++++++++ Multiprocessing test/pool_test.py | 20 ++ Multiprocessing test/thread_priority_queue.py | 59 ++++ Multiprocessing test/tkinter_mp.py | 18 + 8 files changed, 515 insertions(+) create mode 100644 Multiprocessing test/async_multiprocessing_test.py create mode 100644 Multiprocessing test/mp_counter.py create mode 100644 Multiprocessing test/mp_test.py create mode 100644 Multiprocessing test/multiprocessing create mode 100644 Multiprocessing test/multiprocessing_test.py create mode 100644 Multiprocessing test/pool_test.py create mode 100644 Multiprocessing test/thread_priority_queue.py create mode 100644 Multiprocessing test/tkinter_mp.py diff --git a/Multiprocessing test/async_multiprocessing_test.py b/Multiprocessing test/async_multiprocessing_test.py new file mode 100644 index 0000000..6d03b72 --- /dev/null +++ b/Multiprocessing test/async_multiprocessing_test.py @@ -0,0 +1,34 @@ +import multiprocessing as mp +import random +import string + +random.seed(123) + +# Define an output queue +output = mp.Queue() + +# define a example function +def rand_string(length, output): + """ Generates a random string of numbers, lower- and uppercase chars. """ + rand_str = ''.join(random.choice( + string.ascii_lowercase + + string.ascii_uppercase + + string.digits) + for i in range(length)) + output.put(rand_str) + +# Setup a list of processes that we want to run +processes = [mp.Process(target=rand_string, args=(5, output)) for x in range(4)] + +# Run processes +for p in processes: + p.start() + +# Exit the completed processes +for p in processes: + p.join() + +# Get process results from the output queue +results = [output.get() for p in processes] + +print(results) diff --git a/Multiprocessing test/mp_counter.py b/Multiprocessing test/mp_counter.py new file mode 100644 index 0000000..b4b112c --- /dev/null +++ b/Multiprocessing test/mp_counter.py @@ -0,0 +1,29 @@ +import time +from multiprocessing import Process, Value, Lock + +class Counter(object): + def __init__(self, initval=0): + self.val = Value('i', initval) + self.lock = Lock() + + def increment(self): + with self.lock: + self.val.value += 1 + + def value(self): + with self.lock: + return self.val.value + +def func(counter): + for i in range(50): + time.sleep(0.01) + counter.increment() + +if __name__ == '__main__': + counter = Counter(0) + procs = [Process(target=func, args=(counter,)) for i in range(10)] + + for p in procs: p.start() + for p in procs: p.join() + + print (counter.value()) \ No newline at end of file diff --git a/Multiprocessing test/mp_test.py b/Multiprocessing test/mp_test.py new file mode 100644 index 0000000..742fe5c --- /dev/null +++ b/Multiprocessing test/mp_test.py @@ -0,0 +1,40 @@ +import tkinter as tk +from tkinter import ttk +import tkinter.messagebox + +import time # to simulate long-running function + +from threading import Thread + +class RedactionSolutions: + + def __init__(self, master): + tk.Button(master, text="Run RS Wizard", fg='green', command=self.run).grid(row=5) + + def long_running_function(self): + #self.working = tk.Label(root, text='In Progress. Please Wait...') + self.working = ttk.Progressbar(root, orient='horizontal', mode='determinate') + self.working.config(value=0, maximum=10) + self.working.grid(row=6, columnspan=2) + + for x in range(10): + print(x) + self.working.config(value=x) + time.sleep(1) + + restart = tkinter.messagebox.askquestion('RS Wizard', 'Redaction complete! See file location for result.\nWould you like to redact another file?') + if restart == 'yes': + #self.reset() + self.working.grid_forget() + else: + root.destroy() + + def run(self): + Thread(target=self.long_running_function).start() + + + +root = tk.Tk() +root.wm_title("RS Wizard") +RedactionSolutions(root) +root.mainloop() \ No newline at end of file diff --git a/Multiprocessing test/multiprocessing b/Multiprocessing test/multiprocessing new file mode 100644 index 0000000..2a5eb46 --- /dev/null +++ b/Multiprocessing test/multiprocessing @@ -0,0 +1 @@ +/data_2/Projects/h2inc/multiprocessing_test.py \ No newline at end of file diff --git a/Multiprocessing test/multiprocessing_test.py b/Multiprocessing test/multiprocessing_test.py new file mode 100644 index 0000000..748c70c --- /dev/null +++ b/Multiprocessing test/multiprocessing_test.py @@ -0,0 +1,314 @@ +#!/usr/bin/env python3.5 + +# You are free to use and/or change this code for +# your own needs. + +# Original code (c)2018 Jan Lerking +# Program to test various scenarios of +# single thread, multi thread, pool and process + +import os +import sys +import multiprocessing as mp +import time +import io +import cpuinfo +import queue +import threading +import time +import random +import string +import shutil +from tkinter import Tk, ttk, Label, Button, LabelFrame +from tkinter import filedialog, Entry, Checkbutton +from tkinter import Grid, StringVar, DoubleVar +from tkinter import N, E, S, W +from tkinter import DISABLED, NORMAL +from os import errno +import os +from os.path import expanduser +import platform +class file_progress: + def __init__(self, master): + self.sourcedir = StringVar() + self.destdir = StringVar() + self.addinc = StringVar() + self.cfilevar = DoubleVar() + self.totalvar = 0 + self.filecnt = 0 + self.infofolder = 'Number of folders: 0' + self.infofile = 'Number of headers: 0' + self.currentfile = 'Current file: ' + self.totprogress = 'Total progress: ' + self.sourcedir.set('Select source directory!') + self.destdir.set('Select destination directory!') + + self.master = master + self.master.title('File progress') + self.master.grid_columnconfigure(1, weight=1) + + self.frame = LabelFrame(master, text='Progress') + self.frame.grid(row=0, column=0, columnspan=3, sticky=N+S+E+W, padx=5, pady=5) + self.frame.grid_columnconfigure(1, weight=1) + + self.totallabel = Label(self.frame, text=self.totprogress) + self.totallabel.grid(row=8, column=0, sticky=W, padx=5, pady=5) + self.totallabel.config(state=DISABLED) + + self.totalprogress = ttk.Progressbar(self.frame, orient='horizontal', mode='determinate') + self.totalprogress.grid(row=9, column=0, columnspan=3, sticky=N+S+E+W, padx=5, pady=5) + self.totalprogress.config(variable=self.totalvar, maximum=self.filecnt) + + def select_sourcedir(self, sourcedir): + root.directory = os.path.abspath(filedialog.askdirectory()) + if root.directory: + sourcedir.set(root.directory) + filecnt = sourcedir_filecnt(root.directory) + if filecnt > 0: + tempstr = 'Number of headers: '+str(filecnt) + temptot = 'Total progress: 0 of '+str(filecnt) + print ('Source directory: ', sourcedir.get()) + self.destlabel.config(state=NORMAL) + self.destentry.config(state=NORMAL) + self.destdir_button.config(state=NORMAL) + self.infofiles.config(text=tempstr) + self.totallabel.config(text=temptot) + self.filecnt = filecnt + foldercnt = sourcedir_foldercnt(root.directory) + if foldercnt > 0: + tempstr = 'Number of folders: '+str(foldercnt) + self.infofolders.config(text=tempstr) + + def select_destdir(self, destdir): + root.directory = filedialog.askdirectory() + if root.directory: + destdir.set(root.directory) + print ('Destination directory: ', destdir.get()) + self.incchkbox.config(state=NORMAL) + self.infofolders.config(state=NORMAL) + self.infofiles.config(state=NORMAL) + self.translate_button.config(state=NORMAL) + self.cfilelabel.config(state=NORMAL) + self.totallabel.config(state=NORMAL) + + def translate(self, destdir, sourcedir, addinc): + doinc = addinc.get() + dest = destdir.get() + source = sourcedir.get() + if doinc == 'yes': + dest = dest+'/include' + print(os.path.exists(os.path.dirname(dest))) + if not os.path.exists(os.path.dirname(dest)): + try: + os.makedirs(os.path.dirname(dest)) + except OSError as exc: # Guard against race condition + if exc.errno != errno.EEXIST: + raise + destdir.set(dest) + print ('Destination directory: ', destdir.get()) + process_files(source, dest) + + def cfileprogress_update(cnt): + self.cfilevar = cnt + + def currentfile_update(current): + self.currentfile = 'Current file: '+current + +root = Tk() +root.update() +#root.minsize(350, 210) +#width = (root.winfo_screenwidth()/2)-(350/2) +#height = (root.winfo_screenheight()/2)-(210/2) +#root.geometry('+%d+%d' % (width, height)) +root.resizable(False, False) +f_progress = file_progress(root) +root.mainloop() + +cpu_info = cpuinfo.get_cpu_info() +num_cores = mp.cpu_count() +print(num_cores) + +filelist = [] +exitFlag = 0 +queueLock = threading.Lock() +workQueue = queue.Queue(0) +threads = [] +class myThread (threading.Thread): + def __init__(self, threadID, name, q): + threading.Thread.__init__(self) + self.threadID = threadID + self.name = name + self.q = q + def run(self): + print ("Starting " + self.name) + process_data(self.name, self.q) + print ("Exiting " + self.name) + +def process_data(threadName, q): + while not exitFlag: + queueLock.acquire() + if not workQueue.empty(): + data = q.get() + process_file(data) + queueLock.release() + print ("%s processing %s" % (threadName, data)) + else: + queueLock.release() + time.sleep(1) + +def sourcedir_filecnt(sourcedir): + ### Return the number of files, ending with '.h', in sourcedir - including subdirectories ### + cnt = 0 + global filelist + for folderName, subfolders, files in os.walk(sourcedir): + for file in files: + if file.lower().endswith('.h'): + cnt += 1 + filelist += [folderName+'/'+file] + return cnt + +def process_files(sourcedir, destdir): + global filelist + for f in filelist: + process_file(f) + +def process_file(data): + outfile = '' + inputfile = data + encodings = ['utf-8', 'latin-1', 'windows-1250', 'windows-1252', 'ascii', + 'big5', 'big5hkscs', 'cp037', 'cp273', 'cp424', 'cp437', 'cp500', + 'cp720', 'cp737', 'cp775', 'cp850', 'cp852', 'cp855', 'cp856', + 'cp857', 'cp858', 'cp860', 'cp861', 'cp862', 'cp863', 'cp864', 'cp865', + 'cp866', 'cp869', 'cp874', 'cp875', 'cp932', 'cp949', 'cp950', 'cp1006', + 'cp1026', 'cp1125', 'cp1140', 'cp1250', 'cp1251', 'cp1252', 'cp1253', 'cp1254', + 'cp1255', 'cp1256', 'cp1257', 'cp1258', 'cp65001', 'euc-jp', 'euc-jis-2004', + 'euc-jisx0213', 'euc-kr', 'gb2312', 'gbk', 'gb18030', 'hz', 'iso2022-jp', + 'iso2022-jp-1', 'iso2022-jp-2', 'iso2022-jp-2004', 'iso2022-jp-3', + 'iso2022-jp-ext', 'iso2022-kr', 'iso8859-2', 'iso8859-3', 'iso8859-4', + 'iso8859-5', 'iso8859-6', 'iso8859-7', 'iso8859-8', 'iso8859-9', 'iso8859-10', + 'iso8859-11', 'iso8859-13', 'iso8859-14', 'iso8859-15', 'iso8859-16', 'johab', + 'koi8-r', 'koi8-t', 'koi8-u', 'kz1048', 'mac-cyrillic', 'mac-greek', + 'mac-iceland', 'mac-latin2', 'mac-roman', 'mac-turkish', 'ptcp154', + 'shift-jis', 'shift-jis-2004', 'shift-jisx0213', 'utf-32', 'utf-32-be', + 'utf-32-le', 'utf-16', 'utf-16-be', 'utf-16-le', 'utf-7', 'utf-8-sig'] + for e in encodings: + try: + fh = io.open(data, 'r', encoding=e) + fh.readlines() + fh.seek(0) + except UnicodeDecodeError: + print('got unicode error with %s , trying different encoding' % e) + else: + #print('opening the file with encoding: %s ' % e) + break + #print(os.path.basename(data)) + for lines in fh: + outfile = outfile+lines + fh.close() + outputfile = os.path.splitext(inputfile)[0]+'.inc' + outputfile = str(outputfile).replace(sourcedir, destdir) + #print(outputfile) + if not os.path.exists(os.path.dirname(outputfile)): + try: + os.makedirs(os.path.dirname(outputfile)) + except OSError as exc: # Guard against race condition + if exc.errno != errno.EEXIST: + raise + newfile = open(outputfile, "w") + newfile.write(outfile) + newfile.close() + +def single_thread(): + t1 = time.time() + cnt = sourcedir_filecnt(sourcedir) + print('Files in: '+str(cnt)) + process_files(sourcedir, destdir) + print('Single thread process time: '+str(time.time()-t1)) + +def multi_thread(): + global filelist + global exitFlag + + t1 = time.time() + cnt = sourcedir_filecnt(sourcedir) + threadList = ["thread 1", "thread 2", "thread 3", "thread 4", "thread 5"] + nameList = filelist + threadID = 0 + + # Create new threads + for tName in threadList: + thread = myThread(threadID, tName, workQueue) + thread.start() + threads.append(thread) + threadID += 1 + + # Fill the queue + queueLock.acquire() + for word in nameList: + workQueue.put(word) + queueLock.release() + + # Wait for queue to empty + while not workQueue.empty(): + pass + + # Notify threads it's time to exit + exitFlag = 1 + + # Wait for all threads to complete + for t in threads: + t.join() + print ("Exiting Main Thread") + print('Files in: '+str(cnt)) + print('Multi thread process time: '+str(time.time()-t1)) + +def async_process(num): + t1 = time.time() + cnt = sourcedir_filecnt(sourcedir) + + pool = mp.Pool(processes=num) + #[pool.apply_async(process_file, args=(f,)) for f in filelist] + pool.map(process_file, filelist) + #output = [p.get() for p in results] + #print(output) + + # Setup a list of processes that we want to run + #processes = [mp.Process(target=rand_string, args=(5, output)) for x in range(8)] + #processes = [mp.Process(target=process_files, args=(sourcedir, destdir)) for x in range(num)] + + # Run processes + #for p in processes: + #p.start() + + # Exit the completed processes + #for p in processes: + #p.join() + + # Get process results from the output queue + #results = [output.get() for p in processes] + + #print(results) + print('Files in: '+str(cnt)) + print('Multi ('+str(num)+') processes time: '+str(time.time()-t1)) + +def cleanup(dest): + shutil.rmtree(dest) + +#sourcedir = 'C:/Users/dksojlg/Documents/gtk+-3.22.26' +sourcedir = '/usr/include' +#destdir = 'C:/Users/dksojlg/Documents/include' +destdir = '/data_2/include' + +#print(cpu_info) +#single_thread() #2543 files - Single thread process time: 1.3732633590698242 sec. +#os.rmdir(destdir) +#multi_thread() #2543 files - Syncronized 5 threads process time: 80.25179100036621 sec. +#os.rmdir(destdir) +async_process(16) #2543 files - 2 processes, process time: 1.2379400730133057 sec. +#cleanup(destdir) +#async_process(4) #2543 files - 2 processes, process time: 2.6622860431671143 sec. +#cleanup(destdir) +#async_process(6) #2543 files - 2 processes, process time: 2.6622860431671143 sec. +#cleanup(destdir) +#async_process(8) #2543 files - 2 processes, process time: 2.6622860431671143 sec. +#cleanup(destdir) diff --git a/Multiprocessing test/pool_test.py b/Multiprocessing test/pool_test.py new file mode 100644 index 0000000..d7db443 --- /dev/null +++ b/Multiprocessing test/pool_test.py @@ -0,0 +1,20 @@ +from multiprocessing import Pool + +import time + +work = (["A", 5], ["B", 2], ["C", 1], ["D", 3]) + + +def work_log(work_data): + print(" Process %s waiting %s seconds" % (work_data[0], work_data[1])) + time.sleep(int(work_data[1])) + print(" Process %s Finished." % work_data[0]) + + +def pool_handler(): + p = Pool(2) + p.map(work_log, work) + + +if __name__ == '__main__': + pool_handler() \ No newline at end of file diff --git a/Multiprocessing test/thread_priority_queue.py b/Multiprocessing test/thread_priority_queue.py new file mode 100644 index 0000000..08daa57 --- /dev/null +++ b/Multiprocessing test/thread_priority_queue.py @@ -0,0 +1,59 @@ +from Queue import Queue +import threading +import time + +exitFlag = 0 + +class myThread (threading.Thread): + def __init__(self, threadID, name, q): + threading.Thread.__init__(self) + self.threadID = threadID + self.name = name + self.q = q + def run(self): + print ("Starting " + self.name) + process_data(self.name, self.q) + print ("Exiting " + self.name) + +def process_data(threadName, q): + while not exitFlag: + queueLock.acquire() + if not workQueue.empty(): + data = q.get() + queueLock.release() + print ("%s processing %s" % (threadName, data)) + else: + queueLock.release() + time.sleep(1) + +threadList = ["Thread-1", "Thread-2", "Thread-3"] +nameList = ["One", "Two", "Three", "Four", "Five"] +queueLock = threading.Lock() +workQueue = Queue.Queue(10) +threads = [] +threadID = 1 + +# Create new threads +for tName in threadList: + thread = myThread(threadID, tName, workQueue) + thread.start() + threads.append(thread) + threadID += 1 + +# Fill the queue +queueLock.acquire() +for word in nameList: + workQueue.put(word) +queueLock.release() + +# Wait for queue to empty +while not workQueue.empty(): + pass + +# Notify threads it's time to exit +exitFlag = 1 + +# Wait for all threads to complete +for t in threads: + t.join() +print ("Exiting Main Thread") \ No newline at end of file diff --git a/Multiprocessing test/tkinter_mp.py b/Multiprocessing test/tkinter_mp.py new file mode 100644 index 0000000..f132443 --- /dev/null +++ b/Multiprocessing test/tkinter_mp.py @@ -0,0 +1,18 @@ +from multiprocessing import Process, Value, Lock +from tkinter import Tk +from tkinter import Tk, ttk, Label, Button, LabelFrame +from tkinter import filedialog, Entry, Checkbutton +from tkinter import Grid, StringVar, DoubleVar +from tkinter import N, E, S, W +from tkinter import DISABLED, NORMAL + +class testGUI: + def __init__(self, master, initval=0): + self.progress = Value('i', initval) + self.lock = Lock() + + def read_dir(dirname): + + def get_directories(self): + p = Process(target=read_dir, args=dirname) + p.start() \ No newline at end of file From 879eeb15c17d184f921745b5b4202a17f7dc27ea Mon Sep 17 00:00:00 2001 From: Lerking <33354709+Lerking@users.noreply.github.com> Date: Tue, 20 Feb 2018 11:31:58 +0100 Subject: [PATCH 10/37] Add files via upload --- .../async_multiprocessing_test.py | 34 ++ .../Multiprocessing test/calc_pi.py | 163 +++++++++ .../Multiprocessing test/calc_pi_linux.py | 133 ++++++++ .../Multiprocessing test/calc_pi_win.py | 134 ++++++++ .../Multiprocessing test/gui_mp.py | 43 +++ .../Multiprocessing test/mp_counter.py | 29 ++ .../Multiprocessing test/mp_test.py | 40 +++ .../Multiprocessing test/multiprocessing | 1 + .../multiprocessing_test.py | 314 ++++++++++++++++++ .../Multiprocessing test/pool_test.py | 20 ++ .../thread_priority_queue.py | 59 ++++ .../Multiprocessing test/tkinter_mp.py | 57 ++++ 12 files changed, 1027 insertions(+) create mode 100644 Multiprocessing test/Multiprocessing test/async_multiprocessing_test.py create mode 100644 Multiprocessing test/Multiprocessing test/calc_pi.py create mode 100644 Multiprocessing test/Multiprocessing test/calc_pi_linux.py create mode 100644 Multiprocessing test/Multiprocessing test/calc_pi_win.py create mode 100644 Multiprocessing test/Multiprocessing test/gui_mp.py create mode 100644 Multiprocessing test/Multiprocessing test/mp_counter.py create mode 100644 Multiprocessing test/Multiprocessing test/mp_test.py create mode 100644 Multiprocessing test/Multiprocessing test/multiprocessing create mode 100644 Multiprocessing test/Multiprocessing test/multiprocessing_test.py create mode 100644 Multiprocessing test/Multiprocessing test/pool_test.py create mode 100644 Multiprocessing test/Multiprocessing test/thread_priority_queue.py create mode 100644 Multiprocessing test/Multiprocessing test/tkinter_mp.py diff --git a/Multiprocessing test/Multiprocessing test/async_multiprocessing_test.py b/Multiprocessing test/Multiprocessing test/async_multiprocessing_test.py new file mode 100644 index 0000000..6d03b72 --- /dev/null +++ b/Multiprocessing test/Multiprocessing test/async_multiprocessing_test.py @@ -0,0 +1,34 @@ +import multiprocessing as mp +import random +import string + +random.seed(123) + +# Define an output queue +output = mp.Queue() + +# define a example function +def rand_string(length, output): + """ Generates a random string of numbers, lower- and uppercase chars. """ + rand_str = ''.join(random.choice( + string.ascii_lowercase + + string.ascii_uppercase + + string.digits) + for i in range(length)) + output.put(rand_str) + +# Setup a list of processes that we want to run +processes = [mp.Process(target=rand_string, args=(5, output)) for x in range(4)] + +# Run processes +for p in processes: + p.start() + +# Exit the completed processes +for p in processes: + p.join() + +# Get process results from the output queue +results = [output.get() for p in processes] + +print(results) diff --git a/Multiprocessing test/Multiprocessing test/calc_pi.py b/Multiprocessing test/Multiprocessing test/calc_pi.py new file mode 100644 index 0000000..b56848e --- /dev/null +++ b/Multiprocessing test/Multiprocessing test/calc_pi.py @@ -0,0 +1,163 @@ +#!/usr/bin/env python3 + +GUI_WIN = 0 +GUI_LINUX = 1 + +import sys +if sys.platform == 'win32': + GUI = GUI_WIN +else: + GUI = GUI_LINUX + +from tkinter import (Tk, BOTH, Text, E, W, S, N, END, + NORMAL, DISABLED, StringVar) +from tkinter.ttk import Frame, Label, Button, Progressbar, Entry +from tkinter import scrolledtext + +if GUI == GUI_WIN: + from multiprocessing import Process, Manager, Queue + from queue import Empty +else: + from multiprocessing import Queue, Process + import queue + +from decimal import Decimal, getcontext + +DELAY1 = 80 +DELAY2 = 20 + +if GUI == GUI_WIN: + # Queue must be global + q = Queue() + +class commonUI(Frame): + def __init__(self, parent): + Frame.__init__(self, parent) + self.parent = parent + self.initUI() + + def initUI(self): + + self.parent.title("Pi computation") + self.pack(fill=BOTH, expand=True) + + self.grid_columnconfigure(4, weight=1) + self.grid_rowconfigure(3, weight=1) + + lbl1 = Label(self, text="Digits:") + lbl1.grid(row=0, column=0, sticky=E, padx=10, pady=10) + + self.ent1 = Entry(self, width=10) + self.ent1.insert(END, "4000") + self.ent1.grid(row=0, column=1, sticky=W) + + lbl2 = Label(self, text="Accuracy:") + lbl2.grid(row=0, column=2, sticky=E, padx=10, pady=10) + + self.ent2 = Entry(self, width=10) + self.ent2.insert(END, "100") + self.ent2.grid(row=0, column=3, sticky=W) + + self.startBtn = Button(self, text="Start", command=self.onStart) + self.startBtn.grid(row=1, column=0, padx=10, pady=5, sticky=W) + + self.pbar = Progressbar(self, mode='indeterminate') + self.pbar.grid(row=1, column=1, columnspan=3, sticky=W+E) + + self.txt = scrolledtext.ScrolledText(self) + self.txt.grid(row=2, column=0, rowspan=4, padx=10, pady=5, columnspan=5, sticky=E+W+S+N) + + def onStart(self): + + self.startBtn.config(state=DISABLED) + self.txt.delete("1.0", END) + + digits = int(self.ent1.get()) + accuracy = int(self.ent2.get()) + + self.p1 = Process(target=generatePi, args=(q, digits, accuracy)) + self.p1.start() + self.pbar.start(DELAY2) + self.after(DELAY1, self.onGetValue) + + + def onGetValue(self): + + if (self.p1.is_alive()): + + self.after(DELAY1, self.onGetValue) + return + else: + + try: + + self.txt.insert('end', q.get(0)) + self.txt.insert('end', "\n") + self.pbar.stop() + self.startBtn.config(state=NORMAL) + + except Empty: + print("queue is empty") + +class win_Example(Frame): + + def __init__(self, parent): + Frame.__init__(self, parent, name="frame") + commonUI(parent) + +if GUI == GUI_WIN: + # Generate function must be a top-level module funtion + def generatePi(q, digs, acc): + + getcontext().prec = digs + + pi = Decimal(0) + k = 0 + n = acc + + while k < n: + pi += (Decimal(1)/(16**k))*((Decimal(4)/(8*k+1)) - \ + (Decimal(2)/(8*k+4)) - (Decimal(1)/(8*k+5))- \ + (Decimal(1)/(8*k+6))) + k += 1 + + q.put(pi) + +class linux_Example(Frame): + + def __init__(self, parent, q): + Frame.__init__(self, parent) + + self.queue = q + commonUI(parent) + + def generatePi(self, queue): + + getcontext().prec = self.digits + + pi = Decimal(0) + k = 0 + n = self.accuracy + + while k < n: + pi += (Decimal(1)/(16**k))*((Decimal(4)/(8*k+1)) - \ + (Decimal(2)/(8*k+4)) - (Decimal(1)/(8*k+5))- \ + (Decimal(1)/(8*k+6))) + k += 1 + print (self.p1.is_alive()) + + queue.put(pi) + print("end") + +def main(): + if GUI == GUI_LINUX: + q = Queue() + + root = Tk() + root.geometry("400x350+300+300") + app = win_Example(root) if GUI == GUI_WIN else linux_Example(root) + root.mainloop() + + +if __name__ == '__main__': + main() diff --git a/Multiprocessing test/Multiprocessing test/calc_pi_linux.py b/Multiprocessing test/Multiprocessing test/calc_pi_linux.py new file mode 100644 index 0000000..b6ef69d --- /dev/null +++ b/Multiprocessing test/Multiprocessing test/calc_pi_linux.py @@ -0,0 +1,133 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +""" +ZetCode Tkinter e-book + +This script produces a long-running task of calculating +a large Pi number, while keeping the GUI responsive. + +Author: Jan Bodnar +Last modified: January 2016 +Website: www.zetcode.com +""" + +from tkinter import (Tk, BOTH, Text, E, W, S, N, END, + NORMAL, DISABLED, StringVar) +from tkinter.ttk import Frame, Label, Button, Progressbar, Entry +from tkinter import scrolledtext + +from multiprocessing import Queue, Process +import queue +from decimal import Decimal, getcontext + +DELAY1 = 80 +DELAY2 = 20 + +class Example(Frame): + + def __init__(self, parent, q): + Frame.__init__(self, parent) + + self.queue = q + self.parent = parent + self.initUI() + + + def initUI(self): + + self.parent.title("Pi computation") + self.pack(fill=BOTH, expand=True) + + self.grid_columnconfigure(4, weight=1) + self.grid_rowconfigure(3, weight=1) + + lbl1 = Label(self, text="Digits:") + lbl1.grid(row=0, column=0, sticky=E, padx=10, pady=10) + + self.ent1 = Entry(self, width=10) + self.ent1.insert(END, "4000") + self.ent1.grid(row=0, column=1, sticky=W) + + lbl2 = Label(self, text="Accuracy:") + lbl2.grid(row=0, column=2, sticky=E, padx=10, pady=10) + + self.ent2 = Entry(self, width=10) + self.ent2.insert(END, "100") + self.ent2.grid(row=0, column=3, sticky=W) + + self.startBtn = Button(self, text="Start", + command=self.onStart) + self.startBtn.grid(row=1, column=0, padx=10, pady=5, sticky=W) + + self.pbar = Progressbar(self, mode='indeterminate') + self.pbar.grid(row=1, column=1, columnspan=3, sticky=W+E) + + self.txt = scrolledtext.ScrolledText(self) + self.txt.grid(row=2, column=0, rowspan=4, padx=10, pady=5, + columnspan=5, sticky=E+W+S+N) + + + def onStart(self): + + self.startBtn.config(state=DISABLED) + self.txt.delete("1.0", END) + + self.digits = int(self.ent1.get()) + self.accuracy = int(self.ent2.get()) + + self.p1 = Process(target=self.generatePi, args=(self.queue,)) + self.p1.start() + self.pbar.start(DELAY2) + self.after(DELAY1, self.onGetValue) + + + def onGetValue(self): + + if (self.p1.is_alive()): + + self.after(DELAY1, self.onGetValue) + return + else: + + try: + self.txt.insert('end', self.queue.get(0)) + self.txt.insert('end', "\n") + self.pbar.stop() + self.startBtn.config(state=NORMAL) + + except queue.Empty: + print("queue is empty") + + + def generatePi(self, queue): + + getcontext().prec = self.digits + + pi = Decimal(0) + k = 0 + n = self.accuracy + + while k < n: + pi += (Decimal(1)/(16**k))*((Decimal(4)/(8*k+1)) - \ + (Decimal(2)/(8*k+4)) - (Decimal(1)/(8*k+5))- \ + (Decimal(1)/(8*k+6))) + k += 1 + print (self.p1.is_alive()) + + queue.put(pi) + print("end") + + +def main(): + + q = Queue() + + root = Tk() + root.geometry("400x350+300+300") + app = Example(root, q) + root.mainloop() + + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/Multiprocessing test/Multiprocessing test/calc_pi_win.py b/Multiprocessing test/Multiprocessing test/calc_pi_win.py new file mode 100644 index 0000000..b712964 --- /dev/null +++ b/Multiprocessing test/Multiprocessing test/calc_pi_win.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +""" +ZetCode Tkinter e-book + +This script produces a long-running task of calculating +a large Pi number, while keeping the GUI responsive. +This is an example written for Windows. + +Author: Jan Bodnar +Last modified: January 2016 +Website: www.zetcode.com +""" + +from tkinter import (Tk, BOTH, Text, E, W, S, N, END, + NORMAL, DISABLED, StringVar) +from tkinter.ttk import Frame, Label, Button, Progressbar, Entry +from tkinter import scrolledtext + +from multiprocessing import Process, Manager, Queue +from queue import Empty +from decimal import Decimal, getcontext + + +DELAY1 = 80 +DELAY2 = 20 + +# Queue must be global +q = Queue() + +class Example(Frame): + + def __init__(self, parent): + Frame.__init__(self, parent, name="frame") + + self.parent = parent + self.initUI() + + + def initUI(self): + + self.parent.title("Pi computation") + self.pack(fill=BOTH, expand=True) + + self.grid_columnconfigure(4, weight=1) + self.grid_rowconfigure(3, weight=1) + + lbl1 = Label(self, text="Digits:") + lbl1.grid(row=0, column=0, sticky=E, padx=10, pady=10) + + self.ent1 = Entry(self, width=10) + self.ent1.insert(END, "4000") + self.ent1.grid(row=0, column=1, sticky=W) + + lbl2 = Label(self, text="Accuracy:") + lbl2.grid(row=0, column=2, sticky=E, padx=10, pady=10) + + self.ent2 = Entry(self, width=10) + self.ent2.insert(END, "100") + self.ent2.grid(row=0, column=3, sticky=W) + + self.startBtn = Button(self, text="Start", + command=self.onStart) + self.startBtn.grid(row=1, column=0, padx=10, pady=5, sticky=W) + + self.pbar = Progressbar(self, mode='indeterminate') + self.pbar.grid(row=1, column=1, columnspan=3, sticky=W+E) + + self.txt = scrolledtext.ScrolledText(self) + self.txt.grid(row=2, column=0, rowspan=4, padx=10, pady=5, + columnspan=5, sticky=E+W+S+N) + + + def onStart(self): + + self.startBtn.config(state=DISABLED) + self.txt.delete("1.0", END) + + digits = int(self.ent1.get()) + accuracy = int(self.ent2.get()) + + self.p1 = Process(target=generatePi, args=(q, digits, accuracy)) + self.p1.start() + self.pbar.start(DELAY2) + self.after(DELAY1, self.onGetValue) + + + def onGetValue(self): + + if (self.p1.is_alive()): + + self.after(DELAY1, self.onGetValue) + return + else: + + try: + + self.txt.insert('end', q.get(0)) + self.txt.insert('end', "\n") + self.pbar.stop() + self.startBtn.config(state=NORMAL) + + except Empty: + print("queue is empty") + +# Generate function must be a top-level module funtion +def generatePi(q, digs, acc): + + getcontext().prec = digs + + pi = Decimal(0) + k = 0 + n = acc + + while k < n: + pi += (Decimal(1)/(16**k))*((Decimal(4)/(8*k+1)) - \ + (Decimal(2)/(8*k+4)) - (Decimal(1)/(8*k+5))- \ + (Decimal(1)/(8*k+6))) + k += 1 + + q.put(pi) + + +def main(): + + root = Tk() + root.geometry("400x350+300+300") + app = Example(root) + root.mainloop() + + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/Multiprocessing test/Multiprocessing test/gui_mp.py b/Multiprocessing test/Multiprocessing test/gui_mp.py new file mode 100644 index 0000000..01001b0 --- /dev/null +++ b/Multiprocessing test/Multiprocessing test/gui_mp.py @@ -0,0 +1,43 @@ +# Test Code for Tkinter with threads +import tkinter as Tk +import multiprocessing +from Queue import Empty, Full +import time + +class GuiApp(object): + def __init__(self,q): + self.root = Tk.Tk() + self.root.geometry('300x100') + self.text_wid = Tk.Text(self.root,height=100,width=100) + self.text_wid.pack(expand=1,fill=Tk.BOTH) + self.root.after(100,self.CheckQueuePoll,q) + + def CheckQueuePoll(self,c_queue): + try: + str = c_queue.get(0) + self.text_wid.insert('end',str) + except Empty: + pass + finally: + self.root.after(100, self.CheckQueuePoll, c_queue) + +# Data Generator which will generate Data +def GenerateData(q): + for i in range(10): + print("Generating Some Data, Iteration %s" %(i)) + time.sleep(2) + q.put("Some Data from iteration %s \n" %(i)) + + +if __name__ == '__main__': +# Queue which will be used for storing Data + + q = multiprocessing.Queue() + q.cancel_join_thread() # or else thread that puts data will not term + gui = GuiApp(q) + t1 = multiprocessing.Process(target=GenerateData,args=(q,)) + t1.start() + gui.root.mainloop() + + t1.join() + t2.join() \ No newline at end of file diff --git a/Multiprocessing test/Multiprocessing test/mp_counter.py b/Multiprocessing test/Multiprocessing test/mp_counter.py new file mode 100644 index 0000000..4dac468 --- /dev/null +++ b/Multiprocessing test/Multiprocessing test/mp_counter.py @@ -0,0 +1,29 @@ +import time +from multiprocessing import Process, Value, Lock + +class Counter(object): + def __init__(self, initval=0): + self.val = Value('i', initval) + self.lock = Lock() + + def increment(self): + with self.lock: + self.val.value += 1 + + def value(self): + with self.lock: + return self.val.value + +def func(counter): + for i in range(500): + time.sleep(0.001) + counter.increment() + +if __name__ == '__main__': + counter = Counter(0) + procs = [Process(target=func, args=(counter,)) for i in range(10)] + + for p in procs: p.start() + for p in procs: p.join() + + print (counter.value()) \ No newline at end of file diff --git a/Multiprocessing test/Multiprocessing test/mp_test.py b/Multiprocessing test/Multiprocessing test/mp_test.py new file mode 100644 index 0000000..742fe5c --- /dev/null +++ b/Multiprocessing test/Multiprocessing test/mp_test.py @@ -0,0 +1,40 @@ +import tkinter as tk +from tkinter import ttk +import tkinter.messagebox + +import time # to simulate long-running function + +from threading import Thread + +class RedactionSolutions: + + def __init__(self, master): + tk.Button(master, text="Run RS Wizard", fg='green', command=self.run).grid(row=5) + + def long_running_function(self): + #self.working = tk.Label(root, text='In Progress. Please Wait...') + self.working = ttk.Progressbar(root, orient='horizontal', mode='determinate') + self.working.config(value=0, maximum=10) + self.working.grid(row=6, columnspan=2) + + for x in range(10): + print(x) + self.working.config(value=x) + time.sleep(1) + + restart = tkinter.messagebox.askquestion('RS Wizard', 'Redaction complete! See file location for result.\nWould you like to redact another file?') + if restart == 'yes': + #self.reset() + self.working.grid_forget() + else: + root.destroy() + + def run(self): + Thread(target=self.long_running_function).start() + + + +root = tk.Tk() +root.wm_title("RS Wizard") +RedactionSolutions(root) +root.mainloop() \ No newline at end of file diff --git a/Multiprocessing test/Multiprocessing test/multiprocessing b/Multiprocessing test/Multiprocessing test/multiprocessing new file mode 100644 index 0000000..2a5eb46 --- /dev/null +++ b/Multiprocessing test/Multiprocessing test/multiprocessing @@ -0,0 +1 @@ +/data_2/Projects/h2inc/multiprocessing_test.py \ No newline at end of file diff --git a/Multiprocessing test/Multiprocessing test/multiprocessing_test.py b/Multiprocessing test/Multiprocessing test/multiprocessing_test.py new file mode 100644 index 0000000..748c70c --- /dev/null +++ b/Multiprocessing test/Multiprocessing test/multiprocessing_test.py @@ -0,0 +1,314 @@ +#!/usr/bin/env python3.5 + +# You are free to use and/or change this code for +# your own needs. + +# Original code (c)2018 Jan Lerking +# Program to test various scenarios of +# single thread, multi thread, pool and process + +import os +import sys +import multiprocessing as mp +import time +import io +import cpuinfo +import queue +import threading +import time +import random +import string +import shutil +from tkinter import Tk, ttk, Label, Button, LabelFrame +from tkinter import filedialog, Entry, Checkbutton +from tkinter import Grid, StringVar, DoubleVar +from tkinter import N, E, S, W +from tkinter import DISABLED, NORMAL +from os import errno +import os +from os.path import expanduser +import platform +class file_progress: + def __init__(self, master): + self.sourcedir = StringVar() + self.destdir = StringVar() + self.addinc = StringVar() + self.cfilevar = DoubleVar() + self.totalvar = 0 + self.filecnt = 0 + self.infofolder = 'Number of folders: 0' + self.infofile = 'Number of headers: 0' + self.currentfile = 'Current file: ' + self.totprogress = 'Total progress: ' + self.sourcedir.set('Select source directory!') + self.destdir.set('Select destination directory!') + + self.master = master + self.master.title('File progress') + self.master.grid_columnconfigure(1, weight=1) + + self.frame = LabelFrame(master, text='Progress') + self.frame.grid(row=0, column=0, columnspan=3, sticky=N+S+E+W, padx=5, pady=5) + self.frame.grid_columnconfigure(1, weight=1) + + self.totallabel = Label(self.frame, text=self.totprogress) + self.totallabel.grid(row=8, column=0, sticky=W, padx=5, pady=5) + self.totallabel.config(state=DISABLED) + + self.totalprogress = ttk.Progressbar(self.frame, orient='horizontal', mode='determinate') + self.totalprogress.grid(row=9, column=0, columnspan=3, sticky=N+S+E+W, padx=5, pady=5) + self.totalprogress.config(variable=self.totalvar, maximum=self.filecnt) + + def select_sourcedir(self, sourcedir): + root.directory = os.path.abspath(filedialog.askdirectory()) + if root.directory: + sourcedir.set(root.directory) + filecnt = sourcedir_filecnt(root.directory) + if filecnt > 0: + tempstr = 'Number of headers: '+str(filecnt) + temptot = 'Total progress: 0 of '+str(filecnt) + print ('Source directory: ', sourcedir.get()) + self.destlabel.config(state=NORMAL) + self.destentry.config(state=NORMAL) + self.destdir_button.config(state=NORMAL) + self.infofiles.config(text=tempstr) + self.totallabel.config(text=temptot) + self.filecnt = filecnt + foldercnt = sourcedir_foldercnt(root.directory) + if foldercnt > 0: + tempstr = 'Number of folders: '+str(foldercnt) + self.infofolders.config(text=tempstr) + + def select_destdir(self, destdir): + root.directory = filedialog.askdirectory() + if root.directory: + destdir.set(root.directory) + print ('Destination directory: ', destdir.get()) + self.incchkbox.config(state=NORMAL) + self.infofolders.config(state=NORMAL) + self.infofiles.config(state=NORMAL) + self.translate_button.config(state=NORMAL) + self.cfilelabel.config(state=NORMAL) + self.totallabel.config(state=NORMAL) + + def translate(self, destdir, sourcedir, addinc): + doinc = addinc.get() + dest = destdir.get() + source = sourcedir.get() + if doinc == 'yes': + dest = dest+'/include' + print(os.path.exists(os.path.dirname(dest))) + if not os.path.exists(os.path.dirname(dest)): + try: + os.makedirs(os.path.dirname(dest)) + except OSError as exc: # Guard against race condition + if exc.errno != errno.EEXIST: + raise + destdir.set(dest) + print ('Destination directory: ', destdir.get()) + process_files(source, dest) + + def cfileprogress_update(cnt): + self.cfilevar = cnt + + def currentfile_update(current): + self.currentfile = 'Current file: '+current + +root = Tk() +root.update() +#root.minsize(350, 210) +#width = (root.winfo_screenwidth()/2)-(350/2) +#height = (root.winfo_screenheight()/2)-(210/2) +#root.geometry('+%d+%d' % (width, height)) +root.resizable(False, False) +f_progress = file_progress(root) +root.mainloop() + +cpu_info = cpuinfo.get_cpu_info() +num_cores = mp.cpu_count() +print(num_cores) + +filelist = [] +exitFlag = 0 +queueLock = threading.Lock() +workQueue = queue.Queue(0) +threads = [] +class myThread (threading.Thread): + def __init__(self, threadID, name, q): + threading.Thread.__init__(self) + self.threadID = threadID + self.name = name + self.q = q + def run(self): + print ("Starting " + self.name) + process_data(self.name, self.q) + print ("Exiting " + self.name) + +def process_data(threadName, q): + while not exitFlag: + queueLock.acquire() + if not workQueue.empty(): + data = q.get() + process_file(data) + queueLock.release() + print ("%s processing %s" % (threadName, data)) + else: + queueLock.release() + time.sleep(1) + +def sourcedir_filecnt(sourcedir): + ### Return the number of files, ending with '.h', in sourcedir - including subdirectories ### + cnt = 0 + global filelist + for folderName, subfolders, files in os.walk(sourcedir): + for file in files: + if file.lower().endswith('.h'): + cnt += 1 + filelist += [folderName+'/'+file] + return cnt + +def process_files(sourcedir, destdir): + global filelist + for f in filelist: + process_file(f) + +def process_file(data): + outfile = '' + inputfile = data + encodings = ['utf-8', 'latin-1', 'windows-1250', 'windows-1252', 'ascii', + 'big5', 'big5hkscs', 'cp037', 'cp273', 'cp424', 'cp437', 'cp500', + 'cp720', 'cp737', 'cp775', 'cp850', 'cp852', 'cp855', 'cp856', + 'cp857', 'cp858', 'cp860', 'cp861', 'cp862', 'cp863', 'cp864', 'cp865', + 'cp866', 'cp869', 'cp874', 'cp875', 'cp932', 'cp949', 'cp950', 'cp1006', + 'cp1026', 'cp1125', 'cp1140', 'cp1250', 'cp1251', 'cp1252', 'cp1253', 'cp1254', + 'cp1255', 'cp1256', 'cp1257', 'cp1258', 'cp65001', 'euc-jp', 'euc-jis-2004', + 'euc-jisx0213', 'euc-kr', 'gb2312', 'gbk', 'gb18030', 'hz', 'iso2022-jp', + 'iso2022-jp-1', 'iso2022-jp-2', 'iso2022-jp-2004', 'iso2022-jp-3', + 'iso2022-jp-ext', 'iso2022-kr', 'iso8859-2', 'iso8859-3', 'iso8859-4', + 'iso8859-5', 'iso8859-6', 'iso8859-7', 'iso8859-8', 'iso8859-9', 'iso8859-10', + 'iso8859-11', 'iso8859-13', 'iso8859-14', 'iso8859-15', 'iso8859-16', 'johab', + 'koi8-r', 'koi8-t', 'koi8-u', 'kz1048', 'mac-cyrillic', 'mac-greek', + 'mac-iceland', 'mac-latin2', 'mac-roman', 'mac-turkish', 'ptcp154', + 'shift-jis', 'shift-jis-2004', 'shift-jisx0213', 'utf-32', 'utf-32-be', + 'utf-32-le', 'utf-16', 'utf-16-be', 'utf-16-le', 'utf-7', 'utf-8-sig'] + for e in encodings: + try: + fh = io.open(data, 'r', encoding=e) + fh.readlines() + fh.seek(0) + except UnicodeDecodeError: + print('got unicode error with %s , trying different encoding' % e) + else: + #print('opening the file with encoding: %s ' % e) + break + #print(os.path.basename(data)) + for lines in fh: + outfile = outfile+lines + fh.close() + outputfile = os.path.splitext(inputfile)[0]+'.inc' + outputfile = str(outputfile).replace(sourcedir, destdir) + #print(outputfile) + if not os.path.exists(os.path.dirname(outputfile)): + try: + os.makedirs(os.path.dirname(outputfile)) + except OSError as exc: # Guard against race condition + if exc.errno != errno.EEXIST: + raise + newfile = open(outputfile, "w") + newfile.write(outfile) + newfile.close() + +def single_thread(): + t1 = time.time() + cnt = sourcedir_filecnt(sourcedir) + print('Files in: '+str(cnt)) + process_files(sourcedir, destdir) + print('Single thread process time: '+str(time.time()-t1)) + +def multi_thread(): + global filelist + global exitFlag + + t1 = time.time() + cnt = sourcedir_filecnt(sourcedir) + threadList = ["thread 1", "thread 2", "thread 3", "thread 4", "thread 5"] + nameList = filelist + threadID = 0 + + # Create new threads + for tName in threadList: + thread = myThread(threadID, tName, workQueue) + thread.start() + threads.append(thread) + threadID += 1 + + # Fill the queue + queueLock.acquire() + for word in nameList: + workQueue.put(word) + queueLock.release() + + # Wait for queue to empty + while not workQueue.empty(): + pass + + # Notify threads it's time to exit + exitFlag = 1 + + # Wait for all threads to complete + for t in threads: + t.join() + print ("Exiting Main Thread") + print('Files in: '+str(cnt)) + print('Multi thread process time: '+str(time.time()-t1)) + +def async_process(num): + t1 = time.time() + cnt = sourcedir_filecnt(sourcedir) + + pool = mp.Pool(processes=num) + #[pool.apply_async(process_file, args=(f,)) for f in filelist] + pool.map(process_file, filelist) + #output = [p.get() for p in results] + #print(output) + + # Setup a list of processes that we want to run + #processes = [mp.Process(target=rand_string, args=(5, output)) for x in range(8)] + #processes = [mp.Process(target=process_files, args=(sourcedir, destdir)) for x in range(num)] + + # Run processes + #for p in processes: + #p.start() + + # Exit the completed processes + #for p in processes: + #p.join() + + # Get process results from the output queue + #results = [output.get() for p in processes] + + #print(results) + print('Files in: '+str(cnt)) + print('Multi ('+str(num)+') processes time: '+str(time.time()-t1)) + +def cleanup(dest): + shutil.rmtree(dest) + +#sourcedir = 'C:/Users/dksojlg/Documents/gtk+-3.22.26' +sourcedir = '/usr/include' +#destdir = 'C:/Users/dksojlg/Documents/include' +destdir = '/data_2/include' + +#print(cpu_info) +#single_thread() #2543 files - Single thread process time: 1.3732633590698242 sec. +#os.rmdir(destdir) +#multi_thread() #2543 files - Syncronized 5 threads process time: 80.25179100036621 sec. +#os.rmdir(destdir) +async_process(16) #2543 files - 2 processes, process time: 1.2379400730133057 sec. +#cleanup(destdir) +#async_process(4) #2543 files - 2 processes, process time: 2.6622860431671143 sec. +#cleanup(destdir) +#async_process(6) #2543 files - 2 processes, process time: 2.6622860431671143 sec. +#cleanup(destdir) +#async_process(8) #2543 files - 2 processes, process time: 2.6622860431671143 sec. +#cleanup(destdir) diff --git a/Multiprocessing test/Multiprocessing test/pool_test.py b/Multiprocessing test/Multiprocessing test/pool_test.py new file mode 100644 index 0000000..d7db443 --- /dev/null +++ b/Multiprocessing test/Multiprocessing test/pool_test.py @@ -0,0 +1,20 @@ +from multiprocessing import Pool + +import time + +work = (["A", 5], ["B", 2], ["C", 1], ["D", 3]) + + +def work_log(work_data): + print(" Process %s waiting %s seconds" % (work_data[0], work_data[1])) + time.sleep(int(work_data[1])) + print(" Process %s Finished." % work_data[0]) + + +def pool_handler(): + p = Pool(2) + p.map(work_log, work) + + +if __name__ == '__main__': + pool_handler() \ No newline at end of file diff --git a/Multiprocessing test/Multiprocessing test/thread_priority_queue.py b/Multiprocessing test/Multiprocessing test/thread_priority_queue.py new file mode 100644 index 0000000..08daa57 --- /dev/null +++ b/Multiprocessing test/Multiprocessing test/thread_priority_queue.py @@ -0,0 +1,59 @@ +from Queue import Queue +import threading +import time + +exitFlag = 0 + +class myThread (threading.Thread): + def __init__(self, threadID, name, q): + threading.Thread.__init__(self) + self.threadID = threadID + self.name = name + self.q = q + def run(self): + print ("Starting " + self.name) + process_data(self.name, self.q) + print ("Exiting " + self.name) + +def process_data(threadName, q): + while not exitFlag: + queueLock.acquire() + if not workQueue.empty(): + data = q.get() + queueLock.release() + print ("%s processing %s" % (threadName, data)) + else: + queueLock.release() + time.sleep(1) + +threadList = ["Thread-1", "Thread-2", "Thread-3"] +nameList = ["One", "Two", "Three", "Four", "Five"] +queueLock = threading.Lock() +workQueue = Queue.Queue(10) +threads = [] +threadID = 1 + +# Create new threads +for tName in threadList: + thread = myThread(threadID, tName, workQueue) + thread.start() + threads.append(thread) + threadID += 1 + +# Fill the queue +queueLock.acquire() +for word in nameList: + workQueue.put(word) +queueLock.release() + +# Wait for queue to empty +while not workQueue.empty(): + pass + +# Notify threads it's time to exit +exitFlag = 1 + +# Wait for all threads to complete +for t in threads: + t.join() +print ("Exiting Main Thread") \ No newline at end of file diff --git a/Multiprocessing test/Multiprocessing test/tkinter_mp.py b/Multiprocessing test/Multiprocessing test/tkinter_mp.py new file mode 100644 index 0000000..a4a290d --- /dev/null +++ b/Multiprocessing test/Multiprocessing test/tkinter_mp.py @@ -0,0 +1,57 @@ +from multiprocessing import Process, Value, Lock +from tkinter import Tk +from tkinter import Tk, ttk, Label, Button, LabelFrame +from tkinter import Grid, StringVar, DoubleVar +from tkinter import N, E, S, W +from tkinter import DISABLED, NORMAL +import os +import sys + +dirname = 'C:\\Users\\dksojlg\\Documents' + +platform = sys.platform +print(platform) + +class testGUI: + def __init__(self, master, initval=0): + self.progress = Value('i', initval) + self.dircount = Value('i', initval) + self.foldercnt = 'Folders: {}'.format(self.dircount.value) + self.lock = Lock() + + self.master = master + self.master.title('Multiprocess progressbar test!') + self.master.grid_columnconfigure(1, weight=1) + + self.frame = LabelFrame(master, text='Folder count') + self.frame.grid(row=0, column=0, columnspan=3, sticky=N+S+E+W, padx=5, pady=5) + self.frame.grid_columnconfigure(1, weight=1) + + self.sourcelabel = Label(self.frame, text=self.foldercnt) + self.sourcelabel.grid(row=0, column=0, sticky=E, padx=(5, 1), pady=5) + + self.sourcedir_button = Button(self.frame, text="Count folders...", command= lambda: get_directories(self)) + self.sourcedir_button.grid(row=0, column=2, sticky=W, padx=(3, 5), pady=5) + + def increment_dir(self): + with self.lock: + self.dircount.value += 1 + self.sourcelabel.config(text=self.foldercnt) + +def read_dir(testgui, dirname): + for folderName, subfolders, files in os.walk(dirname): + if subfolders: + for subfolder in subfolders: + read_dir(testgui, subfolder) + #testgui.increment_dir() + +def get_directories(testgui): + p = Process(target=read_dir, args=(testgui, dirname,)) + p.start() + +if __name__ == "__main__": + root = Tk() + root.update() + root.resizable(False, False) + testgui = testGUI(root) + root.mainloop() \ No newline at end of file From 500c278d394ab89cfd3319ecbd1c9832704beed6 Mon Sep 17 00:00:00 2001 From: Lerking <33354709+Lerking@users.noreply.github.com> Date: Tue, 20 Feb 2018 11:33:50 +0100 Subject: [PATCH 11/37] Add files via upload --- testing solutions/multiprocessing_test.py | 90 ++++++++++++++++++++++ testing solutions/thread_priority_queue.py | 59 ++++++++++++++ 2 files changed, 149 insertions(+) create mode 100644 testing solutions/multiprocessing_test.py create mode 100644 testing solutions/thread_priority_queue.py diff --git a/testing solutions/multiprocessing_test.py b/testing solutions/multiprocessing_test.py new file mode 100644 index 0000000..59f1e1a --- /dev/null +++ b/testing solutions/multiprocessing_test.py @@ -0,0 +1,90 @@ +# You are free to use and/or change this code for +# your own needs. + +# Original code (c)2018 Jan Lerking +# Program to test various scenarios of +# single thread, multi thread, pool and process + +import os +import sys +import multiprocessing +import time +import io + +num_cores = multiprocessing.cpu_count() +print(num_cores) + +filelist = [] + +def sourcedir_filecnt(sourcedir): + ### Return the number of files, ending with '.h', in sourcedir - including subdirectories ### + cnt = 0 + global filelist + for folderName, subfolders, files in os.walk(sourcedir): + for file in files: + if file.lower().endswith('.h'): + cnt += 1 + filelist += [folderName+'/'+file] + print(folderName+'/'+file) + #print(filelist) + return cnt + +def process_files(sourcedir, destdir): + global filelist + outfile = '' + for f in filelist: + inputfile = f + encodings = ['utf-8', 'latin-1', 'windows-1250', 'windows-1252', 'ascii', + 'big5', 'big5hkscs', 'cp037', 'cp273', 'cp424', 'cp437', 'cp500', + 'cp720', 'cp737', 'cp775', 'cp850', 'cp852', 'cp855', 'cp856', + 'cp857', 'cp858', 'cp860', 'cp861', 'cp862', 'cp863', 'cp864', 'cp865', + 'cp866', 'cp869', 'cp874', 'cp875', 'cp932', 'cp949', 'cp950', 'cp1006', + 'cp1026', 'cp1125', 'cp1140', 'cp1250', 'cp1251', 'cp1252', 'cp1253', 'cp1254', + 'cp1255', 'cp1256', 'cp1257', 'cp1258', 'cp65001', 'euc-jp', 'euc-jis-2004', + 'euc-jisx0213', 'euc-kr', 'gb2312', 'gbk', 'gb18030', 'hz', 'iso2022-jp', + 'iso2022-jp-1', 'iso2022-jp-2', 'iso2022-jp-2004', 'iso2022-jp-3', + 'iso2022-jp-ext', 'iso2022-kr', 'iso8859-2', 'iso8859-3', 'iso8859-4', + 'iso8859-5', 'iso8859-6', 'iso8859-7', 'iso8859-8', 'iso8859-9', 'iso8859-10', + 'iso8859-11', 'iso8859-13', 'iso8859-14', 'iso8859-15', 'iso8859-16', 'johab', + 'koi8-r', 'koi8-t', 'koi8-u', 'kz1048', 'mac-cyrillic', 'mac-greek', + 'mac-iceland', 'mac-latin2', 'mac-roman', 'mac-turkish', 'ptcp154', + 'shift-jis', 'shift-jis-2004', 'shift-jisx0213', 'utf-32', 'utf-32-be', + 'utf-32-le', 'utf-16', 'utf-16-be', 'utf-16-le', 'utf-7', 'utf-8-sig'] + for e in encodings: + try: + fh = io.open(f, 'r', encoding=e) + fh.readlines() + fh.seek(0) + except UnicodeDecodeError: + print('got unicode error with %s , trying different encoding' % e) + else: + print('opening the file with encoding: %s ' % e) + break + print(os.path.basename(f)) + for lines in fh: + print(lines) + outfile = outfile+lines + fh.close() + outputfile = os.path.splitext(inputfile)[0]+'.inc' + outputfile = str(outputfile).replace(sourcedir, destdir) + print(outputfile) + if not os.path.exists(os.path.dirname(outputfile)): + try: + os.makedirs(os.path.dirname(outputfile)) + except OSError as exc: # Guard against race condition + if exc.errno != errno.EEXIST: + raise + newfile = open(outputfile, "w") + newfile.write(outfile) + newfile.close() + +def single_thread(): + t1 = time.time() + sourcedir_filecnt(sourcedir) + process_files(sourcedir, destdir) + print('Single thread process time: '+str(time.time()-t1)) + +sourcedir = 'C:/Users/dksojlg/Documents/gtk+-3.22.26' +destdir = 'C:/Users/dksojlg/Documents/include' + +single_thread() \ No newline at end of file diff --git a/testing solutions/thread_priority_queue.py b/testing solutions/thread_priority_queue.py new file mode 100644 index 0000000..08daa57 --- /dev/null +++ b/testing solutions/thread_priority_queue.py @@ -0,0 +1,59 @@ +from Queue import Queue +import threading +import time + +exitFlag = 0 + +class myThread (threading.Thread): + def __init__(self, threadID, name, q): + threading.Thread.__init__(self) + self.threadID = threadID + self.name = name + self.q = q + def run(self): + print ("Starting " + self.name) + process_data(self.name, self.q) + print ("Exiting " + self.name) + +def process_data(threadName, q): + while not exitFlag: + queueLock.acquire() + if not workQueue.empty(): + data = q.get() + queueLock.release() + print ("%s processing %s" % (threadName, data)) + else: + queueLock.release() + time.sleep(1) + +threadList = ["Thread-1", "Thread-2", "Thread-3"] +nameList = ["One", "Two", "Three", "Four", "Five"] +queueLock = threading.Lock() +workQueue = Queue.Queue(10) +threads = [] +threadID = 1 + +# Create new threads +for tName in threadList: + thread = myThread(threadID, tName, workQueue) + thread.start() + threads.append(thread) + threadID += 1 + +# Fill the queue +queueLock.acquire() +for word in nameList: + workQueue.put(word) +queueLock.release() + +# Wait for queue to empty +while not workQueue.empty(): + pass + +# Notify threads it's time to exit +exitFlag = 1 + +# Wait for all threads to complete +for t in threads: + t.join() +print ("Exiting Main Thread") \ No newline at end of file From a642073d3a5f5c7546604337af29c4f22dc1f7d8 Mon Sep 17 00:00:00 2001 From: Lerking <33354709+Lerking@users.noreply.github.com> Date: Tue, 20 Feb 2018 11:34:52 +0100 Subject: [PATCH 12/37] Add files via upload --- Multiprocessing test/calc_pi.py | 163 ++++++++++++++++++++++++++ Multiprocessing test/calc_pi_linux.py | 133 +++++++++++++++++++++ Multiprocessing test/calc_pi_win.py | 134 +++++++++++++++++++++ Multiprocessing test/gui_mp.py | 43 +++++++ Multiprocessing test/mp_counter.py | 4 +- Multiprocessing test/tkinter_mp.py | 51 +++++++- 6 files changed, 520 insertions(+), 8 deletions(-) create mode 100644 Multiprocessing test/calc_pi.py create mode 100644 Multiprocessing test/calc_pi_linux.py create mode 100644 Multiprocessing test/calc_pi_win.py create mode 100644 Multiprocessing test/gui_mp.py diff --git a/Multiprocessing test/calc_pi.py b/Multiprocessing test/calc_pi.py new file mode 100644 index 0000000..b56848e --- /dev/null +++ b/Multiprocessing test/calc_pi.py @@ -0,0 +1,163 @@ +#!/usr/bin/env python3 + +GUI_WIN = 0 +GUI_LINUX = 1 + +import sys +if sys.platform == 'win32': + GUI = GUI_WIN +else: + GUI = GUI_LINUX + +from tkinter import (Tk, BOTH, Text, E, W, S, N, END, + NORMAL, DISABLED, StringVar) +from tkinter.ttk import Frame, Label, Button, Progressbar, Entry +from tkinter import scrolledtext + +if GUI == GUI_WIN: + from multiprocessing import Process, Manager, Queue + from queue import Empty +else: + from multiprocessing import Queue, Process + import queue + +from decimal import Decimal, getcontext + +DELAY1 = 80 +DELAY2 = 20 + +if GUI == GUI_WIN: + # Queue must be global + q = Queue() + +class commonUI(Frame): + def __init__(self, parent): + Frame.__init__(self, parent) + self.parent = parent + self.initUI() + + def initUI(self): + + self.parent.title("Pi computation") + self.pack(fill=BOTH, expand=True) + + self.grid_columnconfigure(4, weight=1) + self.grid_rowconfigure(3, weight=1) + + lbl1 = Label(self, text="Digits:") + lbl1.grid(row=0, column=0, sticky=E, padx=10, pady=10) + + self.ent1 = Entry(self, width=10) + self.ent1.insert(END, "4000") + self.ent1.grid(row=0, column=1, sticky=W) + + lbl2 = Label(self, text="Accuracy:") + lbl2.grid(row=0, column=2, sticky=E, padx=10, pady=10) + + self.ent2 = Entry(self, width=10) + self.ent2.insert(END, "100") + self.ent2.grid(row=0, column=3, sticky=W) + + self.startBtn = Button(self, text="Start", command=self.onStart) + self.startBtn.grid(row=1, column=0, padx=10, pady=5, sticky=W) + + self.pbar = Progressbar(self, mode='indeterminate') + self.pbar.grid(row=1, column=1, columnspan=3, sticky=W+E) + + self.txt = scrolledtext.ScrolledText(self) + self.txt.grid(row=2, column=0, rowspan=4, padx=10, pady=5, columnspan=5, sticky=E+W+S+N) + + def onStart(self): + + self.startBtn.config(state=DISABLED) + self.txt.delete("1.0", END) + + digits = int(self.ent1.get()) + accuracy = int(self.ent2.get()) + + self.p1 = Process(target=generatePi, args=(q, digits, accuracy)) + self.p1.start() + self.pbar.start(DELAY2) + self.after(DELAY1, self.onGetValue) + + + def onGetValue(self): + + if (self.p1.is_alive()): + + self.after(DELAY1, self.onGetValue) + return + else: + + try: + + self.txt.insert('end', q.get(0)) + self.txt.insert('end', "\n") + self.pbar.stop() + self.startBtn.config(state=NORMAL) + + except Empty: + print("queue is empty") + +class win_Example(Frame): + + def __init__(self, parent): + Frame.__init__(self, parent, name="frame") + commonUI(parent) + +if GUI == GUI_WIN: + # Generate function must be a top-level module funtion + def generatePi(q, digs, acc): + + getcontext().prec = digs + + pi = Decimal(0) + k = 0 + n = acc + + while k < n: + pi += (Decimal(1)/(16**k))*((Decimal(4)/(8*k+1)) - \ + (Decimal(2)/(8*k+4)) - (Decimal(1)/(8*k+5))- \ + (Decimal(1)/(8*k+6))) + k += 1 + + q.put(pi) + +class linux_Example(Frame): + + def __init__(self, parent, q): + Frame.__init__(self, parent) + + self.queue = q + commonUI(parent) + + def generatePi(self, queue): + + getcontext().prec = self.digits + + pi = Decimal(0) + k = 0 + n = self.accuracy + + while k < n: + pi += (Decimal(1)/(16**k))*((Decimal(4)/(8*k+1)) - \ + (Decimal(2)/(8*k+4)) - (Decimal(1)/(8*k+5))- \ + (Decimal(1)/(8*k+6))) + k += 1 + print (self.p1.is_alive()) + + queue.put(pi) + print("end") + +def main(): + if GUI == GUI_LINUX: + q = Queue() + + root = Tk() + root.geometry("400x350+300+300") + app = win_Example(root) if GUI == GUI_WIN else linux_Example(root) + root.mainloop() + + +if __name__ == '__main__': + main() diff --git a/Multiprocessing test/calc_pi_linux.py b/Multiprocessing test/calc_pi_linux.py new file mode 100644 index 0000000..b6ef69d --- /dev/null +++ b/Multiprocessing test/calc_pi_linux.py @@ -0,0 +1,133 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +""" +ZetCode Tkinter e-book + +This script produces a long-running task of calculating +a large Pi number, while keeping the GUI responsive. + +Author: Jan Bodnar +Last modified: January 2016 +Website: www.zetcode.com +""" + +from tkinter import (Tk, BOTH, Text, E, W, S, N, END, + NORMAL, DISABLED, StringVar) +from tkinter.ttk import Frame, Label, Button, Progressbar, Entry +from tkinter import scrolledtext + +from multiprocessing import Queue, Process +import queue +from decimal import Decimal, getcontext + +DELAY1 = 80 +DELAY2 = 20 + +class Example(Frame): + + def __init__(self, parent, q): + Frame.__init__(self, parent) + + self.queue = q + self.parent = parent + self.initUI() + + + def initUI(self): + + self.parent.title("Pi computation") + self.pack(fill=BOTH, expand=True) + + self.grid_columnconfigure(4, weight=1) + self.grid_rowconfigure(3, weight=1) + + lbl1 = Label(self, text="Digits:") + lbl1.grid(row=0, column=0, sticky=E, padx=10, pady=10) + + self.ent1 = Entry(self, width=10) + self.ent1.insert(END, "4000") + self.ent1.grid(row=0, column=1, sticky=W) + + lbl2 = Label(self, text="Accuracy:") + lbl2.grid(row=0, column=2, sticky=E, padx=10, pady=10) + + self.ent2 = Entry(self, width=10) + self.ent2.insert(END, "100") + self.ent2.grid(row=0, column=3, sticky=W) + + self.startBtn = Button(self, text="Start", + command=self.onStart) + self.startBtn.grid(row=1, column=0, padx=10, pady=5, sticky=W) + + self.pbar = Progressbar(self, mode='indeterminate') + self.pbar.grid(row=1, column=1, columnspan=3, sticky=W+E) + + self.txt = scrolledtext.ScrolledText(self) + self.txt.grid(row=2, column=0, rowspan=4, padx=10, pady=5, + columnspan=5, sticky=E+W+S+N) + + + def onStart(self): + + self.startBtn.config(state=DISABLED) + self.txt.delete("1.0", END) + + self.digits = int(self.ent1.get()) + self.accuracy = int(self.ent2.get()) + + self.p1 = Process(target=self.generatePi, args=(self.queue,)) + self.p1.start() + self.pbar.start(DELAY2) + self.after(DELAY1, self.onGetValue) + + + def onGetValue(self): + + if (self.p1.is_alive()): + + self.after(DELAY1, self.onGetValue) + return + else: + + try: + self.txt.insert('end', self.queue.get(0)) + self.txt.insert('end', "\n") + self.pbar.stop() + self.startBtn.config(state=NORMAL) + + except queue.Empty: + print("queue is empty") + + + def generatePi(self, queue): + + getcontext().prec = self.digits + + pi = Decimal(0) + k = 0 + n = self.accuracy + + while k < n: + pi += (Decimal(1)/(16**k))*((Decimal(4)/(8*k+1)) - \ + (Decimal(2)/(8*k+4)) - (Decimal(1)/(8*k+5))- \ + (Decimal(1)/(8*k+6))) + k += 1 + print (self.p1.is_alive()) + + queue.put(pi) + print("end") + + +def main(): + + q = Queue() + + root = Tk() + root.geometry("400x350+300+300") + app = Example(root, q) + root.mainloop() + + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/Multiprocessing test/calc_pi_win.py b/Multiprocessing test/calc_pi_win.py new file mode 100644 index 0000000..b712964 --- /dev/null +++ b/Multiprocessing test/calc_pi_win.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +""" +ZetCode Tkinter e-book + +This script produces a long-running task of calculating +a large Pi number, while keeping the GUI responsive. +This is an example written for Windows. + +Author: Jan Bodnar +Last modified: January 2016 +Website: www.zetcode.com +""" + +from tkinter import (Tk, BOTH, Text, E, W, S, N, END, + NORMAL, DISABLED, StringVar) +from tkinter.ttk import Frame, Label, Button, Progressbar, Entry +from tkinter import scrolledtext + +from multiprocessing import Process, Manager, Queue +from queue import Empty +from decimal import Decimal, getcontext + + +DELAY1 = 80 +DELAY2 = 20 + +# Queue must be global +q = Queue() + +class Example(Frame): + + def __init__(self, parent): + Frame.__init__(self, parent, name="frame") + + self.parent = parent + self.initUI() + + + def initUI(self): + + self.parent.title("Pi computation") + self.pack(fill=BOTH, expand=True) + + self.grid_columnconfigure(4, weight=1) + self.grid_rowconfigure(3, weight=1) + + lbl1 = Label(self, text="Digits:") + lbl1.grid(row=0, column=0, sticky=E, padx=10, pady=10) + + self.ent1 = Entry(self, width=10) + self.ent1.insert(END, "4000") + self.ent1.grid(row=0, column=1, sticky=W) + + lbl2 = Label(self, text="Accuracy:") + lbl2.grid(row=0, column=2, sticky=E, padx=10, pady=10) + + self.ent2 = Entry(self, width=10) + self.ent2.insert(END, "100") + self.ent2.grid(row=0, column=3, sticky=W) + + self.startBtn = Button(self, text="Start", + command=self.onStart) + self.startBtn.grid(row=1, column=0, padx=10, pady=5, sticky=W) + + self.pbar = Progressbar(self, mode='indeterminate') + self.pbar.grid(row=1, column=1, columnspan=3, sticky=W+E) + + self.txt = scrolledtext.ScrolledText(self) + self.txt.grid(row=2, column=0, rowspan=4, padx=10, pady=5, + columnspan=5, sticky=E+W+S+N) + + + def onStart(self): + + self.startBtn.config(state=DISABLED) + self.txt.delete("1.0", END) + + digits = int(self.ent1.get()) + accuracy = int(self.ent2.get()) + + self.p1 = Process(target=generatePi, args=(q, digits, accuracy)) + self.p1.start() + self.pbar.start(DELAY2) + self.after(DELAY1, self.onGetValue) + + + def onGetValue(self): + + if (self.p1.is_alive()): + + self.after(DELAY1, self.onGetValue) + return + else: + + try: + + self.txt.insert('end', q.get(0)) + self.txt.insert('end', "\n") + self.pbar.stop() + self.startBtn.config(state=NORMAL) + + except Empty: + print("queue is empty") + +# Generate function must be a top-level module funtion +def generatePi(q, digs, acc): + + getcontext().prec = digs + + pi = Decimal(0) + k = 0 + n = acc + + while k < n: + pi += (Decimal(1)/(16**k))*((Decimal(4)/(8*k+1)) - \ + (Decimal(2)/(8*k+4)) - (Decimal(1)/(8*k+5))- \ + (Decimal(1)/(8*k+6))) + k += 1 + + q.put(pi) + + +def main(): + + root = Tk() + root.geometry("400x350+300+300") + app = Example(root) + root.mainloop() + + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/Multiprocessing test/gui_mp.py b/Multiprocessing test/gui_mp.py new file mode 100644 index 0000000..01001b0 --- /dev/null +++ b/Multiprocessing test/gui_mp.py @@ -0,0 +1,43 @@ +# Test Code for Tkinter with threads +import tkinter as Tk +import multiprocessing +from Queue import Empty, Full +import time + +class GuiApp(object): + def __init__(self,q): + self.root = Tk.Tk() + self.root.geometry('300x100') + self.text_wid = Tk.Text(self.root,height=100,width=100) + self.text_wid.pack(expand=1,fill=Tk.BOTH) + self.root.after(100,self.CheckQueuePoll,q) + + def CheckQueuePoll(self,c_queue): + try: + str = c_queue.get(0) + self.text_wid.insert('end',str) + except Empty: + pass + finally: + self.root.after(100, self.CheckQueuePoll, c_queue) + +# Data Generator which will generate Data +def GenerateData(q): + for i in range(10): + print("Generating Some Data, Iteration %s" %(i)) + time.sleep(2) + q.put("Some Data from iteration %s \n" %(i)) + + +if __name__ == '__main__': +# Queue which will be used for storing Data + + q = multiprocessing.Queue() + q.cancel_join_thread() # or else thread that puts data will not term + gui = GuiApp(q) + t1 = multiprocessing.Process(target=GenerateData,args=(q,)) + t1.start() + gui.root.mainloop() + + t1.join() + t2.join() \ No newline at end of file diff --git a/Multiprocessing test/mp_counter.py b/Multiprocessing test/mp_counter.py index b4b112c..4dac468 100644 --- a/Multiprocessing test/mp_counter.py +++ b/Multiprocessing test/mp_counter.py @@ -15,8 +15,8 @@ class Counter(object): return self.val.value def func(counter): - for i in range(50): - time.sleep(0.01) + for i in range(500): + time.sleep(0.001) counter.increment() if __name__ == '__main__': diff --git a/Multiprocessing test/tkinter_mp.py b/Multiprocessing test/tkinter_mp.py index f132443..a4a290d 100644 --- a/Multiprocessing test/tkinter_mp.py +++ b/Multiprocessing test/tkinter_mp.py @@ -1,18 +1,57 @@ from multiprocessing import Process, Value, Lock from tkinter import Tk from tkinter import Tk, ttk, Label, Button, LabelFrame -from tkinter import filedialog, Entry, Checkbutton from tkinter import Grid, StringVar, DoubleVar from tkinter import N, E, S, W from tkinter import DISABLED, NORMAL +import os +import sys + +dirname = 'C:\\Users\\dksojlg\\Documents' + +platform = sys.platform +print(platform) class testGUI: def __init__(self, master, initval=0): self.progress = Value('i', initval) + self.dircount = Value('i', initval) + self.foldercnt = 'Folders: {}'.format(self.dircount.value) self.lock = Lock() - def read_dir(dirname): - - def get_directories(self): - p = Process(target=read_dir, args=dirname) - p.start() \ No newline at end of file + self.master = master + self.master.title('Multiprocess progressbar test!') + self.master.grid_columnconfigure(1, weight=1) + + self.frame = LabelFrame(master, text='Folder count') + self.frame.grid(row=0, column=0, columnspan=3, sticky=N+S+E+W, padx=5, pady=5) + self.frame.grid_columnconfigure(1, weight=1) + + self.sourcelabel = Label(self.frame, text=self.foldercnt) + self.sourcelabel.grid(row=0, column=0, sticky=E, padx=(5, 1), pady=5) + + self.sourcedir_button = Button(self.frame, text="Count folders...", command= lambda: get_directories(self)) + self.sourcedir_button.grid(row=0, column=2, sticky=W, padx=(3, 5), pady=5) + + def increment_dir(self): + with self.lock: + self.dircount.value += 1 + self.sourcelabel.config(text=self.foldercnt) + +def read_dir(testgui, dirname): + for folderName, subfolders, files in os.walk(dirname): + if subfolders: + for subfolder in subfolders: + read_dir(testgui, subfolder) + #testgui.increment_dir() + +def get_directories(testgui): + p = Process(target=read_dir, args=(testgui, dirname,)) + p.start() + +if __name__ == "__main__": + root = Tk() + root.update() + root.resizable(False, False) + testgui = testGUI(root) + root.mainloop() \ No newline at end of file From 0a86fa44eb7589a162a115508d3ff0b30f4a7e07 Mon Sep 17 00:00:00 2001 From: Lerking <33354709+Lerking@users.noreply.github.com> Date: Tue, 20 Feb 2018 13:59:09 +0100 Subject: [PATCH 13/37] Add files via upload --- Multiprocessing test/calc_pi.py | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/Multiprocessing test/calc_pi.py b/Multiprocessing test/calc_pi.py index b56848e..5dceeac 100644 --- a/Multiprocessing test/calc_pi.py +++ b/Multiprocessing test/calc_pi.py @@ -15,15 +15,16 @@ from tkinter.ttk import Frame, Label, Button, Progressbar, Entry from tkinter import scrolledtext if GUI == GUI_WIN: - from multiprocessing import Process, Manager, Queue + from multiprocessing import Process, Manager, Queue, Value from queue import Empty else: from multiprocessing import Queue, Process import queue from decimal import Decimal, getcontext +from time import sleep -DELAY1 = 80 +DELAY1 = 20 DELAY2 = 20 if GUI == GUI_WIN: @@ -38,6 +39,10 @@ class commonUI(Frame): def initUI(self): + self.val = Value('i', 0) + self.dig = Value('i', 4000) + self.acc = Value('i', 100) + self.parent.title("Pi computation") self.pack(fill=BOTH, expand=True) @@ -48,22 +53,23 @@ class commonUI(Frame): lbl1.grid(row=0, column=0, sticky=E, padx=10, pady=10) self.ent1 = Entry(self, width=10) - self.ent1.insert(END, "4000") + self.ent1.insert(END, self.dig.value) self.ent1.grid(row=0, column=1, sticky=W) lbl2 = Label(self, text="Accuracy:") lbl2.grid(row=0, column=2, sticky=E, padx=10, pady=10) self.ent2 = Entry(self, width=10) - self.ent2.insert(END, "100") + self.ent2.insert(END, self.acc.value) self.ent2.grid(row=0, column=3, sticky=W) self.startBtn = Button(self, text="Start", command=self.onStart) self.startBtn.grid(row=1, column=0, padx=10, pady=5, sticky=W) - self.pbar = Progressbar(self, mode='indeterminate') + self.pbar = Progressbar(self, mode='determinate') self.pbar.grid(row=1, column=1, columnspan=3, sticky=W+E) - + self.pbar.config(value=0, maximum=self.dig.value) + self.txt = scrolledtext.ScrolledText(self) self.txt.grid(row=2, column=0, rowspan=4, padx=10, pady=5, columnspan=5, sticky=E+W+S+N) @@ -77,14 +83,14 @@ class commonUI(Frame): self.p1 = Process(target=generatePi, args=(q, digits, accuracy)) self.p1.start() - self.pbar.start(DELAY2) self.after(DELAY1, self.onGetValue) def onGetValue(self): if (self.p1.is_alive()): - + self.val.value += 1000 + self.update_pbar() self.after(DELAY1, self.onGetValue) return else: @@ -93,12 +99,14 @@ class commonUI(Frame): self.txt.insert('end', q.get(0)) self.txt.insert('end', "\n") - self.pbar.stop() self.startBtn.config(state=NORMAL) except Empty: print("queue is empty") + def update_pbar(self): + self.pbar.config(value=self.val.value) + class win_Example(Frame): def __init__(self, parent): @@ -120,8 +128,8 @@ if GUI == GUI_WIN: (Decimal(2)/(8*k+4)) - (Decimal(1)/(8*k+5))- \ (Decimal(1)/(8*k+6))) k += 1 - q.put(pi) + time.sleep(1) class linux_Example(Frame): From b6c606f141843ce1788330d5d6c77bb213babf8e Mon Sep 17 00:00:00 2001 From: Jan Lerking Date: Tue, 20 Feb 2018 17:18:43 +0100 Subject: [PATCH 14/37] Cleanup --- .../async_multiprocessing_test.py | 34 -- .../Multiprocessing test/calc_pi.py | 163 --------- .../Multiprocessing test/calc_pi_linux.py | 133 -------- .../Multiprocessing test/calc_pi_win.py | 134 -------- .../Multiprocessing test/gui_mp.py | 43 --- .../Multiprocessing test/mp_counter.py | 29 -- .../Multiprocessing test/mp_test.py | 40 --- .../Multiprocessing test/multiprocessing | 1 - .../multiprocessing_test.py | 314 ------------------ .../Multiprocessing test/pool_test.py | 20 -- .../thread_priority_queue.py | 59 ---- .../Multiprocessing test/tkinter_mp.py | 57 ---- async_multiprocessing_test.py | 34 -- mp_counter.py | 29 -- mp_test.py | 40 --- multiprocessing | 1 - multiprocessing_test.py | 314 ------------------ pool_test.py | 20 -- thread_priority_queue.py | 59 ---- 19 files changed, 1524 deletions(-) delete mode 100644 Multiprocessing test/Multiprocessing test/async_multiprocessing_test.py delete mode 100644 Multiprocessing test/Multiprocessing test/calc_pi.py delete mode 100644 Multiprocessing test/Multiprocessing test/calc_pi_linux.py delete mode 100644 Multiprocessing test/Multiprocessing test/calc_pi_win.py delete mode 100644 Multiprocessing test/Multiprocessing test/gui_mp.py delete mode 100644 Multiprocessing test/Multiprocessing test/mp_counter.py delete mode 100644 Multiprocessing test/Multiprocessing test/mp_test.py delete mode 100644 Multiprocessing test/Multiprocessing test/multiprocessing delete mode 100644 Multiprocessing test/Multiprocessing test/multiprocessing_test.py delete mode 100644 Multiprocessing test/Multiprocessing test/pool_test.py delete mode 100644 Multiprocessing test/Multiprocessing test/thread_priority_queue.py delete mode 100644 Multiprocessing test/Multiprocessing test/tkinter_mp.py delete mode 100644 async_multiprocessing_test.py delete mode 100644 mp_counter.py delete mode 100644 mp_test.py delete mode 120000 multiprocessing delete mode 100755 multiprocessing_test.py delete mode 100644 pool_test.py delete mode 100644 thread_priority_queue.py diff --git a/Multiprocessing test/Multiprocessing test/async_multiprocessing_test.py b/Multiprocessing test/Multiprocessing test/async_multiprocessing_test.py deleted file mode 100644 index 6d03b72..0000000 --- a/Multiprocessing test/Multiprocessing test/async_multiprocessing_test.py +++ /dev/null @@ -1,34 +0,0 @@ -import multiprocessing as mp -import random -import string - -random.seed(123) - -# Define an output queue -output = mp.Queue() - -# define a example function -def rand_string(length, output): - """ Generates a random string of numbers, lower- and uppercase chars. """ - rand_str = ''.join(random.choice( - string.ascii_lowercase - + string.ascii_uppercase - + string.digits) - for i in range(length)) - output.put(rand_str) - -# Setup a list of processes that we want to run -processes = [mp.Process(target=rand_string, args=(5, output)) for x in range(4)] - -# Run processes -for p in processes: - p.start() - -# Exit the completed processes -for p in processes: - p.join() - -# Get process results from the output queue -results = [output.get() for p in processes] - -print(results) diff --git a/Multiprocessing test/Multiprocessing test/calc_pi.py b/Multiprocessing test/Multiprocessing test/calc_pi.py deleted file mode 100644 index b56848e..0000000 --- a/Multiprocessing test/Multiprocessing test/calc_pi.py +++ /dev/null @@ -1,163 +0,0 @@ -#!/usr/bin/env python3 - -GUI_WIN = 0 -GUI_LINUX = 1 - -import sys -if sys.platform == 'win32': - GUI = GUI_WIN -else: - GUI = GUI_LINUX - -from tkinter import (Tk, BOTH, Text, E, W, S, N, END, - NORMAL, DISABLED, StringVar) -from tkinter.ttk import Frame, Label, Button, Progressbar, Entry -from tkinter import scrolledtext - -if GUI == GUI_WIN: - from multiprocessing import Process, Manager, Queue - from queue import Empty -else: - from multiprocessing import Queue, Process - import queue - -from decimal import Decimal, getcontext - -DELAY1 = 80 -DELAY2 = 20 - -if GUI == GUI_WIN: - # Queue must be global - q = Queue() - -class commonUI(Frame): - def __init__(self, parent): - Frame.__init__(self, parent) - self.parent = parent - self.initUI() - - def initUI(self): - - self.parent.title("Pi computation") - self.pack(fill=BOTH, expand=True) - - self.grid_columnconfigure(4, weight=1) - self.grid_rowconfigure(3, weight=1) - - lbl1 = Label(self, text="Digits:") - lbl1.grid(row=0, column=0, sticky=E, padx=10, pady=10) - - self.ent1 = Entry(self, width=10) - self.ent1.insert(END, "4000") - self.ent1.grid(row=0, column=1, sticky=W) - - lbl2 = Label(self, text="Accuracy:") - lbl2.grid(row=0, column=2, sticky=E, padx=10, pady=10) - - self.ent2 = Entry(self, width=10) - self.ent2.insert(END, "100") - self.ent2.grid(row=0, column=3, sticky=W) - - self.startBtn = Button(self, text="Start", command=self.onStart) - self.startBtn.grid(row=1, column=0, padx=10, pady=5, sticky=W) - - self.pbar = Progressbar(self, mode='indeterminate') - self.pbar.grid(row=1, column=1, columnspan=3, sticky=W+E) - - self.txt = scrolledtext.ScrolledText(self) - self.txt.grid(row=2, column=0, rowspan=4, padx=10, pady=5, columnspan=5, sticky=E+W+S+N) - - def onStart(self): - - self.startBtn.config(state=DISABLED) - self.txt.delete("1.0", END) - - digits = int(self.ent1.get()) - accuracy = int(self.ent2.get()) - - self.p1 = Process(target=generatePi, args=(q, digits, accuracy)) - self.p1.start() - self.pbar.start(DELAY2) - self.after(DELAY1, self.onGetValue) - - - def onGetValue(self): - - if (self.p1.is_alive()): - - self.after(DELAY1, self.onGetValue) - return - else: - - try: - - self.txt.insert('end', q.get(0)) - self.txt.insert('end', "\n") - self.pbar.stop() - self.startBtn.config(state=NORMAL) - - except Empty: - print("queue is empty") - -class win_Example(Frame): - - def __init__(self, parent): - Frame.__init__(self, parent, name="frame") - commonUI(parent) - -if GUI == GUI_WIN: - # Generate function must be a top-level module funtion - def generatePi(q, digs, acc): - - getcontext().prec = digs - - pi = Decimal(0) - k = 0 - n = acc - - while k < n: - pi += (Decimal(1)/(16**k))*((Decimal(4)/(8*k+1)) - \ - (Decimal(2)/(8*k+4)) - (Decimal(1)/(8*k+5))- \ - (Decimal(1)/(8*k+6))) - k += 1 - - q.put(pi) - -class linux_Example(Frame): - - def __init__(self, parent, q): - Frame.__init__(self, parent) - - self.queue = q - commonUI(parent) - - def generatePi(self, queue): - - getcontext().prec = self.digits - - pi = Decimal(0) - k = 0 - n = self.accuracy - - while k < n: - pi += (Decimal(1)/(16**k))*((Decimal(4)/(8*k+1)) - \ - (Decimal(2)/(8*k+4)) - (Decimal(1)/(8*k+5))- \ - (Decimal(1)/(8*k+6))) - k += 1 - print (self.p1.is_alive()) - - queue.put(pi) - print("end") - -def main(): - if GUI == GUI_LINUX: - q = Queue() - - root = Tk() - root.geometry("400x350+300+300") - app = win_Example(root) if GUI == GUI_WIN else linux_Example(root) - root.mainloop() - - -if __name__ == '__main__': - main() diff --git a/Multiprocessing test/Multiprocessing test/calc_pi_linux.py b/Multiprocessing test/Multiprocessing test/calc_pi_linux.py deleted file mode 100644 index b6ef69d..0000000 --- a/Multiprocessing test/Multiprocessing test/calc_pi_linux.py +++ /dev/null @@ -1,133 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - -""" -ZetCode Tkinter e-book - -This script produces a long-running task of calculating -a large Pi number, while keeping the GUI responsive. - -Author: Jan Bodnar -Last modified: January 2016 -Website: www.zetcode.com -""" - -from tkinter import (Tk, BOTH, Text, E, W, S, N, END, - NORMAL, DISABLED, StringVar) -from tkinter.ttk import Frame, Label, Button, Progressbar, Entry -from tkinter import scrolledtext - -from multiprocessing import Queue, Process -import queue -from decimal import Decimal, getcontext - -DELAY1 = 80 -DELAY2 = 20 - -class Example(Frame): - - def __init__(self, parent, q): - Frame.__init__(self, parent) - - self.queue = q - self.parent = parent - self.initUI() - - - def initUI(self): - - self.parent.title("Pi computation") - self.pack(fill=BOTH, expand=True) - - self.grid_columnconfigure(4, weight=1) - self.grid_rowconfigure(3, weight=1) - - lbl1 = Label(self, text="Digits:") - lbl1.grid(row=0, column=0, sticky=E, padx=10, pady=10) - - self.ent1 = Entry(self, width=10) - self.ent1.insert(END, "4000") - self.ent1.grid(row=0, column=1, sticky=W) - - lbl2 = Label(self, text="Accuracy:") - lbl2.grid(row=0, column=2, sticky=E, padx=10, pady=10) - - self.ent2 = Entry(self, width=10) - self.ent2.insert(END, "100") - self.ent2.grid(row=0, column=3, sticky=W) - - self.startBtn = Button(self, text="Start", - command=self.onStart) - self.startBtn.grid(row=1, column=0, padx=10, pady=5, sticky=W) - - self.pbar = Progressbar(self, mode='indeterminate') - self.pbar.grid(row=1, column=1, columnspan=3, sticky=W+E) - - self.txt = scrolledtext.ScrolledText(self) - self.txt.grid(row=2, column=0, rowspan=4, padx=10, pady=5, - columnspan=5, sticky=E+W+S+N) - - - def onStart(self): - - self.startBtn.config(state=DISABLED) - self.txt.delete("1.0", END) - - self.digits = int(self.ent1.get()) - self.accuracy = int(self.ent2.get()) - - self.p1 = Process(target=self.generatePi, args=(self.queue,)) - self.p1.start() - self.pbar.start(DELAY2) - self.after(DELAY1, self.onGetValue) - - - def onGetValue(self): - - if (self.p1.is_alive()): - - self.after(DELAY1, self.onGetValue) - return - else: - - try: - self.txt.insert('end', self.queue.get(0)) - self.txt.insert('end', "\n") - self.pbar.stop() - self.startBtn.config(state=NORMAL) - - except queue.Empty: - print("queue is empty") - - - def generatePi(self, queue): - - getcontext().prec = self.digits - - pi = Decimal(0) - k = 0 - n = self.accuracy - - while k < n: - pi += (Decimal(1)/(16**k))*((Decimal(4)/(8*k+1)) - \ - (Decimal(2)/(8*k+4)) - (Decimal(1)/(8*k+5))- \ - (Decimal(1)/(8*k+6))) - k += 1 - print (self.p1.is_alive()) - - queue.put(pi) - print("end") - - -def main(): - - q = Queue() - - root = Tk() - root.geometry("400x350+300+300") - app = Example(root, q) - root.mainloop() - - -if __name__ == '__main__': - main() \ No newline at end of file diff --git a/Multiprocessing test/Multiprocessing test/calc_pi_win.py b/Multiprocessing test/Multiprocessing test/calc_pi_win.py deleted file mode 100644 index b712964..0000000 --- a/Multiprocessing test/Multiprocessing test/calc_pi_win.py +++ /dev/null @@ -1,134 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - -""" -ZetCode Tkinter e-book - -This script produces a long-running task of calculating -a large Pi number, while keeping the GUI responsive. -This is an example written for Windows. - -Author: Jan Bodnar -Last modified: January 2016 -Website: www.zetcode.com -""" - -from tkinter import (Tk, BOTH, Text, E, W, S, N, END, - NORMAL, DISABLED, StringVar) -from tkinter.ttk import Frame, Label, Button, Progressbar, Entry -from tkinter import scrolledtext - -from multiprocessing import Process, Manager, Queue -from queue import Empty -from decimal import Decimal, getcontext - - -DELAY1 = 80 -DELAY2 = 20 - -# Queue must be global -q = Queue() - -class Example(Frame): - - def __init__(self, parent): - Frame.__init__(self, parent, name="frame") - - self.parent = parent - self.initUI() - - - def initUI(self): - - self.parent.title("Pi computation") - self.pack(fill=BOTH, expand=True) - - self.grid_columnconfigure(4, weight=1) - self.grid_rowconfigure(3, weight=1) - - lbl1 = Label(self, text="Digits:") - lbl1.grid(row=0, column=0, sticky=E, padx=10, pady=10) - - self.ent1 = Entry(self, width=10) - self.ent1.insert(END, "4000") - self.ent1.grid(row=0, column=1, sticky=W) - - lbl2 = Label(self, text="Accuracy:") - lbl2.grid(row=0, column=2, sticky=E, padx=10, pady=10) - - self.ent2 = Entry(self, width=10) - self.ent2.insert(END, "100") - self.ent2.grid(row=0, column=3, sticky=W) - - self.startBtn = Button(self, text="Start", - command=self.onStart) - self.startBtn.grid(row=1, column=0, padx=10, pady=5, sticky=W) - - self.pbar = Progressbar(self, mode='indeterminate') - self.pbar.grid(row=1, column=1, columnspan=3, sticky=W+E) - - self.txt = scrolledtext.ScrolledText(self) - self.txt.grid(row=2, column=0, rowspan=4, padx=10, pady=5, - columnspan=5, sticky=E+W+S+N) - - - def onStart(self): - - self.startBtn.config(state=DISABLED) - self.txt.delete("1.0", END) - - digits = int(self.ent1.get()) - accuracy = int(self.ent2.get()) - - self.p1 = Process(target=generatePi, args=(q, digits, accuracy)) - self.p1.start() - self.pbar.start(DELAY2) - self.after(DELAY1, self.onGetValue) - - - def onGetValue(self): - - if (self.p1.is_alive()): - - self.after(DELAY1, self.onGetValue) - return - else: - - try: - - self.txt.insert('end', q.get(0)) - self.txt.insert('end', "\n") - self.pbar.stop() - self.startBtn.config(state=NORMAL) - - except Empty: - print("queue is empty") - -# Generate function must be a top-level module funtion -def generatePi(q, digs, acc): - - getcontext().prec = digs - - pi = Decimal(0) - k = 0 - n = acc - - while k < n: - pi += (Decimal(1)/(16**k))*((Decimal(4)/(8*k+1)) - \ - (Decimal(2)/(8*k+4)) - (Decimal(1)/(8*k+5))- \ - (Decimal(1)/(8*k+6))) - k += 1 - - q.put(pi) - - -def main(): - - root = Tk() - root.geometry("400x350+300+300") - app = Example(root) - root.mainloop() - - -if __name__ == '__main__': - main() \ No newline at end of file diff --git a/Multiprocessing test/Multiprocessing test/gui_mp.py b/Multiprocessing test/Multiprocessing test/gui_mp.py deleted file mode 100644 index 01001b0..0000000 --- a/Multiprocessing test/Multiprocessing test/gui_mp.py +++ /dev/null @@ -1,43 +0,0 @@ -# Test Code for Tkinter with threads -import tkinter as Tk -import multiprocessing -from Queue import Empty, Full -import time - -class GuiApp(object): - def __init__(self,q): - self.root = Tk.Tk() - self.root.geometry('300x100') - self.text_wid = Tk.Text(self.root,height=100,width=100) - self.text_wid.pack(expand=1,fill=Tk.BOTH) - self.root.after(100,self.CheckQueuePoll,q) - - def CheckQueuePoll(self,c_queue): - try: - str = c_queue.get(0) - self.text_wid.insert('end',str) - except Empty: - pass - finally: - self.root.after(100, self.CheckQueuePoll, c_queue) - -# Data Generator which will generate Data -def GenerateData(q): - for i in range(10): - print("Generating Some Data, Iteration %s" %(i)) - time.sleep(2) - q.put("Some Data from iteration %s \n" %(i)) - - -if __name__ == '__main__': -# Queue which will be used for storing Data - - q = multiprocessing.Queue() - q.cancel_join_thread() # or else thread that puts data will not term - gui = GuiApp(q) - t1 = multiprocessing.Process(target=GenerateData,args=(q,)) - t1.start() - gui.root.mainloop() - - t1.join() - t2.join() \ No newline at end of file diff --git a/Multiprocessing test/Multiprocessing test/mp_counter.py b/Multiprocessing test/Multiprocessing test/mp_counter.py deleted file mode 100644 index 4dac468..0000000 --- a/Multiprocessing test/Multiprocessing test/mp_counter.py +++ /dev/null @@ -1,29 +0,0 @@ -import time -from multiprocessing import Process, Value, Lock - -class Counter(object): - def __init__(self, initval=0): - self.val = Value('i', initval) - self.lock = Lock() - - def increment(self): - with self.lock: - self.val.value += 1 - - def value(self): - with self.lock: - return self.val.value - -def func(counter): - for i in range(500): - time.sleep(0.001) - counter.increment() - -if __name__ == '__main__': - counter = Counter(0) - procs = [Process(target=func, args=(counter,)) for i in range(10)] - - for p in procs: p.start() - for p in procs: p.join() - - print (counter.value()) \ No newline at end of file diff --git a/Multiprocessing test/Multiprocessing test/mp_test.py b/Multiprocessing test/Multiprocessing test/mp_test.py deleted file mode 100644 index 742fe5c..0000000 --- a/Multiprocessing test/Multiprocessing test/mp_test.py +++ /dev/null @@ -1,40 +0,0 @@ -import tkinter as tk -from tkinter import ttk -import tkinter.messagebox - -import time # to simulate long-running function - -from threading import Thread - -class RedactionSolutions: - - def __init__(self, master): - tk.Button(master, text="Run RS Wizard", fg='green', command=self.run).grid(row=5) - - def long_running_function(self): - #self.working = tk.Label(root, text='In Progress. Please Wait...') - self.working = ttk.Progressbar(root, orient='horizontal', mode='determinate') - self.working.config(value=0, maximum=10) - self.working.grid(row=6, columnspan=2) - - for x in range(10): - print(x) - self.working.config(value=x) - time.sleep(1) - - restart = tkinter.messagebox.askquestion('RS Wizard', 'Redaction complete! See file location for result.\nWould you like to redact another file?') - if restart == 'yes': - #self.reset() - self.working.grid_forget() - else: - root.destroy() - - def run(self): - Thread(target=self.long_running_function).start() - - - -root = tk.Tk() -root.wm_title("RS Wizard") -RedactionSolutions(root) -root.mainloop() \ No newline at end of file diff --git a/Multiprocessing test/Multiprocessing test/multiprocessing b/Multiprocessing test/Multiprocessing test/multiprocessing deleted file mode 100644 index 2a5eb46..0000000 --- a/Multiprocessing test/Multiprocessing test/multiprocessing +++ /dev/null @@ -1 +0,0 @@ -/data_2/Projects/h2inc/multiprocessing_test.py \ No newline at end of file diff --git a/Multiprocessing test/Multiprocessing test/multiprocessing_test.py b/Multiprocessing test/Multiprocessing test/multiprocessing_test.py deleted file mode 100644 index 748c70c..0000000 --- a/Multiprocessing test/Multiprocessing test/multiprocessing_test.py +++ /dev/null @@ -1,314 +0,0 @@ -#!/usr/bin/env python3.5 - -# You are free to use and/or change this code for -# your own needs. - -# Original code (c)2018 Jan Lerking -# Program to test various scenarios of -# single thread, multi thread, pool and process - -import os -import sys -import multiprocessing as mp -import time -import io -import cpuinfo -import queue -import threading -import time -import random -import string -import shutil -from tkinter import Tk, ttk, Label, Button, LabelFrame -from tkinter import filedialog, Entry, Checkbutton -from tkinter import Grid, StringVar, DoubleVar -from tkinter import N, E, S, W -from tkinter import DISABLED, NORMAL -from os import errno -import os -from os.path import expanduser -import platform -class file_progress: - def __init__(self, master): - self.sourcedir = StringVar() - self.destdir = StringVar() - self.addinc = StringVar() - self.cfilevar = DoubleVar() - self.totalvar = 0 - self.filecnt = 0 - self.infofolder = 'Number of folders: 0' - self.infofile = 'Number of headers: 0' - self.currentfile = 'Current file: ' - self.totprogress = 'Total progress: ' - self.sourcedir.set('Select source directory!') - self.destdir.set('Select destination directory!') - - self.master = master - self.master.title('File progress') - self.master.grid_columnconfigure(1, weight=1) - - self.frame = LabelFrame(master, text='Progress') - self.frame.grid(row=0, column=0, columnspan=3, sticky=N+S+E+W, padx=5, pady=5) - self.frame.grid_columnconfigure(1, weight=1) - - self.totallabel = Label(self.frame, text=self.totprogress) - self.totallabel.grid(row=8, column=0, sticky=W, padx=5, pady=5) - self.totallabel.config(state=DISABLED) - - self.totalprogress = ttk.Progressbar(self.frame, orient='horizontal', mode='determinate') - self.totalprogress.grid(row=9, column=0, columnspan=3, sticky=N+S+E+W, padx=5, pady=5) - self.totalprogress.config(variable=self.totalvar, maximum=self.filecnt) - - def select_sourcedir(self, sourcedir): - root.directory = os.path.abspath(filedialog.askdirectory()) - if root.directory: - sourcedir.set(root.directory) - filecnt = sourcedir_filecnt(root.directory) - if filecnt > 0: - tempstr = 'Number of headers: '+str(filecnt) - temptot = 'Total progress: 0 of '+str(filecnt) - print ('Source directory: ', sourcedir.get()) - self.destlabel.config(state=NORMAL) - self.destentry.config(state=NORMAL) - self.destdir_button.config(state=NORMAL) - self.infofiles.config(text=tempstr) - self.totallabel.config(text=temptot) - self.filecnt = filecnt - foldercnt = sourcedir_foldercnt(root.directory) - if foldercnt > 0: - tempstr = 'Number of folders: '+str(foldercnt) - self.infofolders.config(text=tempstr) - - def select_destdir(self, destdir): - root.directory = filedialog.askdirectory() - if root.directory: - destdir.set(root.directory) - print ('Destination directory: ', destdir.get()) - self.incchkbox.config(state=NORMAL) - self.infofolders.config(state=NORMAL) - self.infofiles.config(state=NORMAL) - self.translate_button.config(state=NORMAL) - self.cfilelabel.config(state=NORMAL) - self.totallabel.config(state=NORMAL) - - def translate(self, destdir, sourcedir, addinc): - doinc = addinc.get() - dest = destdir.get() - source = sourcedir.get() - if doinc == 'yes': - dest = dest+'/include' - print(os.path.exists(os.path.dirname(dest))) - if not os.path.exists(os.path.dirname(dest)): - try: - os.makedirs(os.path.dirname(dest)) - except OSError as exc: # Guard against race condition - if exc.errno != errno.EEXIST: - raise - destdir.set(dest) - print ('Destination directory: ', destdir.get()) - process_files(source, dest) - - def cfileprogress_update(cnt): - self.cfilevar = cnt - - def currentfile_update(current): - self.currentfile = 'Current file: '+current - -root = Tk() -root.update() -#root.minsize(350, 210) -#width = (root.winfo_screenwidth()/2)-(350/2) -#height = (root.winfo_screenheight()/2)-(210/2) -#root.geometry('+%d+%d' % (width, height)) -root.resizable(False, False) -f_progress = file_progress(root) -root.mainloop() - -cpu_info = cpuinfo.get_cpu_info() -num_cores = mp.cpu_count() -print(num_cores) - -filelist = [] -exitFlag = 0 -queueLock = threading.Lock() -workQueue = queue.Queue(0) -threads = [] -class myThread (threading.Thread): - def __init__(self, threadID, name, q): - threading.Thread.__init__(self) - self.threadID = threadID - self.name = name - self.q = q - def run(self): - print ("Starting " + self.name) - process_data(self.name, self.q) - print ("Exiting " + self.name) - -def process_data(threadName, q): - while not exitFlag: - queueLock.acquire() - if not workQueue.empty(): - data = q.get() - process_file(data) - queueLock.release() - print ("%s processing %s" % (threadName, data)) - else: - queueLock.release() - time.sleep(1) - -def sourcedir_filecnt(sourcedir): - ### Return the number of files, ending with '.h', in sourcedir - including subdirectories ### - cnt = 0 - global filelist - for folderName, subfolders, files in os.walk(sourcedir): - for file in files: - if file.lower().endswith('.h'): - cnt += 1 - filelist += [folderName+'/'+file] - return cnt - -def process_files(sourcedir, destdir): - global filelist - for f in filelist: - process_file(f) - -def process_file(data): - outfile = '' - inputfile = data - encodings = ['utf-8', 'latin-1', 'windows-1250', 'windows-1252', 'ascii', - 'big5', 'big5hkscs', 'cp037', 'cp273', 'cp424', 'cp437', 'cp500', - 'cp720', 'cp737', 'cp775', 'cp850', 'cp852', 'cp855', 'cp856', - 'cp857', 'cp858', 'cp860', 'cp861', 'cp862', 'cp863', 'cp864', 'cp865', - 'cp866', 'cp869', 'cp874', 'cp875', 'cp932', 'cp949', 'cp950', 'cp1006', - 'cp1026', 'cp1125', 'cp1140', 'cp1250', 'cp1251', 'cp1252', 'cp1253', 'cp1254', - 'cp1255', 'cp1256', 'cp1257', 'cp1258', 'cp65001', 'euc-jp', 'euc-jis-2004', - 'euc-jisx0213', 'euc-kr', 'gb2312', 'gbk', 'gb18030', 'hz', 'iso2022-jp', - 'iso2022-jp-1', 'iso2022-jp-2', 'iso2022-jp-2004', 'iso2022-jp-3', - 'iso2022-jp-ext', 'iso2022-kr', 'iso8859-2', 'iso8859-3', 'iso8859-4', - 'iso8859-5', 'iso8859-6', 'iso8859-7', 'iso8859-8', 'iso8859-9', 'iso8859-10', - 'iso8859-11', 'iso8859-13', 'iso8859-14', 'iso8859-15', 'iso8859-16', 'johab', - 'koi8-r', 'koi8-t', 'koi8-u', 'kz1048', 'mac-cyrillic', 'mac-greek', - 'mac-iceland', 'mac-latin2', 'mac-roman', 'mac-turkish', 'ptcp154', - 'shift-jis', 'shift-jis-2004', 'shift-jisx0213', 'utf-32', 'utf-32-be', - 'utf-32-le', 'utf-16', 'utf-16-be', 'utf-16-le', 'utf-7', 'utf-8-sig'] - for e in encodings: - try: - fh = io.open(data, 'r', encoding=e) - fh.readlines() - fh.seek(0) - except UnicodeDecodeError: - print('got unicode error with %s , trying different encoding' % e) - else: - #print('opening the file with encoding: %s ' % e) - break - #print(os.path.basename(data)) - for lines in fh: - outfile = outfile+lines - fh.close() - outputfile = os.path.splitext(inputfile)[0]+'.inc' - outputfile = str(outputfile).replace(sourcedir, destdir) - #print(outputfile) - if not os.path.exists(os.path.dirname(outputfile)): - try: - os.makedirs(os.path.dirname(outputfile)) - except OSError as exc: # Guard against race condition - if exc.errno != errno.EEXIST: - raise - newfile = open(outputfile, "w") - newfile.write(outfile) - newfile.close() - -def single_thread(): - t1 = time.time() - cnt = sourcedir_filecnt(sourcedir) - print('Files in: '+str(cnt)) - process_files(sourcedir, destdir) - print('Single thread process time: '+str(time.time()-t1)) - -def multi_thread(): - global filelist - global exitFlag - - t1 = time.time() - cnt = sourcedir_filecnt(sourcedir) - threadList = ["thread 1", "thread 2", "thread 3", "thread 4", "thread 5"] - nameList = filelist - threadID = 0 - - # Create new threads - for tName in threadList: - thread = myThread(threadID, tName, workQueue) - thread.start() - threads.append(thread) - threadID += 1 - - # Fill the queue - queueLock.acquire() - for word in nameList: - workQueue.put(word) - queueLock.release() - - # Wait for queue to empty - while not workQueue.empty(): - pass - - # Notify threads it's time to exit - exitFlag = 1 - - # Wait for all threads to complete - for t in threads: - t.join() - print ("Exiting Main Thread") - print('Files in: '+str(cnt)) - print('Multi thread process time: '+str(time.time()-t1)) - -def async_process(num): - t1 = time.time() - cnt = sourcedir_filecnt(sourcedir) - - pool = mp.Pool(processes=num) - #[pool.apply_async(process_file, args=(f,)) for f in filelist] - pool.map(process_file, filelist) - #output = [p.get() for p in results] - #print(output) - - # Setup a list of processes that we want to run - #processes = [mp.Process(target=rand_string, args=(5, output)) for x in range(8)] - #processes = [mp.Process(target=process_files, args=(sourcedir, destdir)) for x in range(num)] - - # Run processes - #for p in processes: - #p.start() - - # Exit the completed processes - #for p in processes: - #p.join() - - # Get process results from the output queue - #results = [output.get() for p in processes] - - #print(results) - print('Files in: '+str(cnt)) - print('Multi ('+str(num)+') processes time: '+str(time.time()-t1)) - -def cleanup(dest): - shutil.rmtree(dest) - -#sourcedir = 'C:/Users/dksojlg/Documents/gtk+-3.22.26' -sourcedir = '/usr/include' -#destdir = 'C:/Users/dksojlg/Documents/include' -destdir = '/data_2/include' - -#print(cpu_info) -#single_thread() #2543 files - Single thread process time: 1.3732633590698242 sec. -#os.rmdir(destdir) -#multi_thread() #2543 files - Syncronized 5 threads process time: 80.25179100036621 sec. -#os.rmdir(destdir) -async_process(16) #2543 files - 2 processes, process time: 1.2379400730133057 sec. -#cleanup(destdir) -#async_process(4) #2543 files - 2 processes, process time: 2.6622860431671143 sec. -#cleanup(destdir) -#async_process(6) #2543 files - 2 processes, process time: 2.6622860431671143 sec. -#cleanup(destdir) -#async_process(8) #2543 files - 2 processes, process time: 2.6622860431671143 sec. -#cleanup(destdir) diff --git a/Multiprocessing test/Multiprocessing test/pool_test.py b/Multiprocessing test/Multiprocessing test/pool_test.py deleted file mode 100644 index d7db443..0000000 --- a/Multiprocessing test/Multiprocessing test/pool_test.py +++ /dev/null @@ -1,20 +0,0 @@ -from multiprocessing import Pool - -import time - -work = (["A", 5], ["B", 2], ["C", 1], ["D", 3]) - - -def work_log(work_data): - print(" Process %s waiting %s seconds" % (work_data[0], work_data[1])) - time.sleep(int(work_data[1])) - print(" Process %s Finished." % work_data[0]) - - -def pool_handler(): - p = Pool(2) - p.map(work_log, work) - - -if __name__ == '__main__': - pool_handler() \ No newline at end of file diff --git a/Multiprocessing test/Multiprocessing test/thread_priority_queue.py b/Multiprocessing test/Multiprocessing test/thread_priority_queue.py deleted file mode 100644 index 08daa57..0000000 --- a/Multiprocessing test/Multiprocessing test/thread_priority_queue.py +++ /dev/null @@ -1,59 +0,0 @@ -from Queue import Queue -import threading -import time - -exitFlag = 0 - -class myThread (threading.Thread): - def __init__(self, threadID, name, q): - threading.Thread.__init__(self) - self.threadID = threadID - self.name = name - self.q = q - def run(self): - print ("Starting " + self.name) - process_data(self.name, self.q) - print ("Exiting " + self.name) - -def process_data(threadName, q): - while not exitFlag: - queueLock.acquire() - if not workQueue.empty(): - data = q.get() - queueLock.release() - print ("%s processing %s" % (threadName, data)) - else: - queueLock.release() - time.sleep(1) - -threadList = ["Thread-1", "Thread-2", "Thread-3"] -nameList = ["One", "Two", "Three", "Four", "Five"] -queueLock = threading.Lock() -workQueue = Queue.Queue(10) -threads = [] -threadID = 1 - -# Create new threads -for tName in threadList: - thread = myThread(threadID, tName, workQueue) - thread.start() - threads.append(thread) - threadID += 1 - -# Fill the queue -queueLock.acquire() -for word in nameList: - workQueue.put(word) -queueLock.release() - -# Wait for queue to empty -while not workQueue.empty(): - pass - -# Notify threads it's time to exit -exitFlag = 1 - -# Wait for all threads to complete -for t in threads: - t.join() -print ("Exiting Main Thread") \ No newline at end of file diff --git a/Multiprocessing test/Multiprocessing test/tkinter_mp.py b/Multiprocessing test/Multiprocessing test/tkinter_mp.py deleted file mode 100644 index a4a290d..0000000 --- a/Multiprocessing test/Multiprocessing test/tkinter_mp.py +++ /dev/null @@ -1,57 +0,0 @@ -from multiprocessing import Process, Value, Lock -from tkinter import Tk -from tkinter import Tk, ttk, Label, Button, LabelFrame -from tkinter import Grid, StringVar, DoubleVar -from tkinter import N, E, S, W -from tkinter import DISABLED, NORMAL -import os -import sys - -dirname = 'C:\\Users\\dksojlg\\Documents' - -platform = sys.platform -print(platform) - -class testGUI: - def __init__(self, master, initval=0): - self.progress = Value('i', initval) - self.dircount = Value('i', initval) - self.foldercnt = 'Folders: {}'.format(self.dircount.value) - self.lock = Lock() - - self.master = master - self.master.title('Multiprocess progressbar test!') - self.master.grid_columnconfigure(1, weight=1) - - self.frame = LabelFrame(master, text='Folder count') - self.frame.grid(row=0, column=0, columnspan=3, sticky=N+S+E+W, padx=5, pady=5) - self.frame.grid_columnconfigure(1, weight=1) - - self.sourcelabel = Label(self.frame, text=self.foldercnt) - self.sourcelabel.grid(row=0, column=0, sticky=E, padx=(5, 1), pady=5) - - self.sourcedir_button = Button(self.frame, text="Count folders...", command= lambda: get_directories(self)) - self.sourcedir_button.grid(row=0, column=2, sticky=W, padx=(3, 5), pady=5) - - def increment_dir(self): - with self.lock: - self.dircount.value += 1 - self.sourcelabel.config(text=self.foldercnt) - -def read_dir(testgui, dirname): - for folderName, subfolders, files in os.walk(dirname): - if subfolders: - for subfolder in subfolders: - read_dir(testgui, subfolder) - #testgui.increment_dir() - -def get_directories(testgui): - p = Process(target=read_dir, args=(testgui, dirname,)) - p.start() - -if __name__ == "__main__": - root = Tk() - root.update() - root.resizable(False, False) - testgui = testGUI(root) - root.mainloop() \ No newline at end of file diff --git a/async_multiprocessing_test.py b/async_multiprocessing_test.py deleted file mode 100644 index 6d03b72..0000000 --- a/async_multiprocessing_test.py +++ /dev/null @@ -1,34 +0,0 @@ -import multiprocessing as mp -import random -import string - -random.seed(123) - -# Define an output queue -output = mp.Queue() - -# define a example function -def rand_string(length, output): - """ Generates a random string of numbers, lower- and uppercase chars. """ - rand_str = ''.join(random.choice( - string.ascii_lowercase - + string.ascii_uppercase - + string.digits) - for i in range(length)) - output.put(rand_str) - -# Setup a list of processes that we want to run -processes = [mp.Process(target=rand_string, args=(5, output)) for x in range(4)] - -# Run processes -for p in processes: - p.start() - -# Exit the completed processes -for p in processes: - p.join() - -# Get process results from the output queue -results = [output.get() for p in processes] - -print(results) diff --git a/mp_counter.py b/mp_counter.py deleted file mode 100644 index b4b112c..0000000 --- a/mp_counter.py +++ /dev/null @@ -1,29 +0,0 @@ -import time -from multiprocessing import Process, Value, Lock - -class Counter(object): - def __init__(self, initval=0): - self.val = Value('i', initval) - self.lock = Lock() - - def increment(self): - with self.lock: - self.val.value += 1 - - def value(self): - with self.lock: - return self.val.value - -def func(counter): - for i in range(50): - time.sleep(0.01) - counter.increment() - -if __name__ == '__main__': - counter = Counter(0) - procs = [Process(target=func, args=(counter,)) for i in range(10)] - - for p in procs: p.start() - for p in procs: p.join() - - print (counter.value()) \ No newline at end of file diff --git a/mp_test.py b/mp_test.py deleted file mode 100644 index 742fe5c..0000000 --- a/mp_test.py +++ /dev/null @@ -1,40 +0,0 @@ -import tkinter as tk -from tkinter import ttk -import tkinter.messagebox - -import time # to simulate long-running function - -from threading import Thread - -class RedactionSolutions: - - def __init__(self, master): - tk.Button(master, text="Run RS Wizard", fg='green', command=self.run).grid(row=5) - - def long_running_function(self): - #self.working = tk.Label(root, text='In Progress. Please Wait...') - self.working = ttk.Progressbar(root, orient='horizontal', mode='determinate') - self.working.config(value=0, maximum=10) - self.working.grid(row=6, columnspan=2) - - for x in range(10): - print(x) - self.working.config(value=x) - time.sleep(1) - - restart = tkinter.messagebox.askquestion('RS Wizard', 'Redaction complete! See file location for result.\nWould you like to redact another file?') - if restart == 'yes': - #self.reset() - self.working.grid_forget() - else: - root.destroy() - - def run(self): - Thread(target=self.long_running_function).start() - - - -root = tk.Tk() -root.wm_title("RS Wizard") -RedactionSolutions(root) -root.mainloop() \ No newline at end of file diff --git a/multiprocessing b/multiprocessing deleted file mode 120000 index 2a5eb46..0000000 --- a/multiprocessing +++ /dev/null @@ -1 +0,0 @@ -/data_2/Projects/h2inc/multiprocessing_test.py \ No newline at end of file diff --git a/multiprocessing_test.py b/multiprocessing_test.py deleted file mode 100755 index 748c70c..0000000 --- a/multiprocessing_test.py +++ /dev/null @@ -1,314 +0,0 @@ -#!/usr/bin/env python3.5 - -# You are free to use and/or change this code for -# your own needs. - -# Original code (c)2018 Jan Lerking -# Program to test various scenarios of -# single thread, multi thread, pool and process - -import os -import sys -import multiprocessing as mp -import time -import io -import cpuinfo -import queue -import threading -import time -import random -import string -import shutil -from tkinter import Tk, ttk, Label, Button, LabelFrame -from tkinter import filedialog, Entry, Checkbutton -from tkinter import Grid, StringVar, DoubleVar -from tkinter import N, E, S, W -from tkinter import DISABLED, NORMAL -from os import errno -import os -from os.path import expanduser -import platform -class file_progress: - def __init__(self, master): - self.sourcedir = StringVar() - self.destdir = StringVar() - self.addinc = StringVar() - self.cfilevar = DoubleVar() - self.totalvar = 0 - self.filecnt = 0 - self.infofolder = 'Number of folders: 0' - self.infofile = 'Number of headers: 0' - self.currentfile = 'Current file: ' - self.totprogress = 'Total progress: ' - self.sourcedir.set('Select source directory!') - self.destdir.set('Select destination directory!') - - self.master = master - self.master.title('File progress') - self.master.grid_columnconfigure(1, weight=1) - - self.frame = LabelFrame(master, text='Progress') - self.frame.grid(row=0, column=0, columnspan=3, sticky=N+S+E+W, padx=5, pady=5) - self.frame.grid_columnconfigure(1, weight=1) - - self.totallabel = Label(self.frame, text=self.totprogress) - self.totallabel.grid(row=8, column=0, sticky=W, padx=5, pady=5) - self.totallabel.config(state=DISABLED) - - self.totalprogress = ttk.Progressbar(self.frame, orient='horizontal', mode='determinate') - self.totalprogress.grid(row=9, column=0, columnspan=3, sticky=N+S+E+W, padx=5, pady=5) - self.totalprogress.config(variable=self.totalvar, maximum=self.filecnt) - - def select_sourcedir(self, sourcedir): - root.directory = os.path.abspath(filedialog.askdirectory()) - if root.directory: - sourcedir.set(root.directory) - filecnt = sourcedir_filecnt(root.directory) - if filecnt > 0: - tempstr = 'Number of headers: '+str(filecnt) - temptot = 'Total progress: 0 of '+str(filecnt) - print ('Source directory: ', sourcedir.get()) - self.destlabel.config(state=NORMAL) - self.destentry.config(state=NORMAL) - self.destdir_button.config(state=NORMAL) - self.infofiles.config(text=tempstr) - self.totallabel.config(text=temptot) - self.filecnt = filecnt - foldercnt = sourcedir_foldercnt(root.directory) - if foldercnt > 0: - tempstr = 'Number of folders: '+str(foldercnt) - self.infofolders.config(text=tempstr) - - def select_destdir(self, destdir): - root.directory = filedialog.askdirectory() - if root.directory: - destdir.set(root.directory) - print ('Destination directory: ', destdir.get()) - self.incchkbox.config(state=NORMAL) - self.infofolders.config(state=NORMAL) - self.infofiles.config(state=NORMAL) - self.translate_button.config(state=NORMAL) - self.cfilelabel.config(state=NORMAL) - self.totallabel.config(state=NORMAL) - - def translate(self, destdir, sourcedir, addinc): - doinc = addinc.get() - dest = destdir.get() - source = sourcedir.get() - if doinc == 'yes': - dest = dest+'/include' - print(os.path.exists(os.path.dirname(dest))) - if not os.path.exists(os.path.dirname(dest)): - try: - os.makedirs(os.path.dirname(dest)) - except OSError as exc: # Guard against race condition - if exc.errno != errno.EEXIST: - raise - destdir.set(dest) - print ('Destination directory: ', destdir.get()) - process_files(source, dest) - - def cfileprogress_update(cnt): - self.cfilevar = cnt - - def currentfile_update(current): - self.currentfile = 'Current file: '+current - -root = Tk() -root.update() -#root.minsize(350, 210) -#width = (root.winfo_screenwidth()/2)-(350/2) -#height = (root.winfo_screenheight()/2)-(210/2) -#root.geometry('+%d+%d' % (width, height)) -root.resizable(False, False) -f_progress = file_progress(root) -root.mainloop() - -cpu_info = cpuinfo.get_cpu_info() -num_cores = mp.cpu_count() -print(num_cores) - -filelist = [] -exitFlag = 0 -queueLock = threading.Lock() -workQueue = queue.Queue(0) -threads = [] -class myThread (threading.Thread): - def __init__(self, threadID, name, q): - threading.Thread.__init__(self) - self.threadID = threadID - self.name = name - self.q = q - def run(self): - print ("Starting " + self.name) - process_data(self.name, self.q) - print ("Exiting " + self.name) - -def process_data(threadName, q): - while not exitFlag: - queueLock.acquire() - if not workQueue.empty(): - data = q.get() - process_file(data) - queueLock.release() - print ("%s processing %s" % (threadName, data)) - else: - queueLock.release() - time.sleep(1) - -def sourcedir_filecnt(sourcedir): - ### Return the number of files, ending with '.h', in sourcedir - including subdirectories ### - cnt = 0 - global filelist - for folderName, subfolders, files in os.walk(sourcedir): - for file in files: - if file.lower().endswith('.h'): - cnt += 1 - filelist += [folderName+'/'+file] - return cnt - -def process_files(sourcedir, destdir): - global filelist - for f in filelist: - process_file(f) - -def process_file(data): - outfile = '' - inputfile = data - encodings = ['utf-8', 'latin-1', 'windows-1250', 'windows-1252', 'ascii', - 'big5', 'big5hkscs', 'cp037', 'cp273', 'cp424', 'cp437', 'cp500', - 'cp720', 'cp737', 'cp775', 'cp850', 'cp852', 'cp855', 'cp856', - 'cp857', 'cp858', 'cp860', 'cp861', 'cp862', 'cp863', 'cp864', 'cp865', - 'cp866', 'cp869', 'cp874', 'cp875', 'cp932', 'cp949', 'cp950', 'cp1006', - 'cp1026', 'cp1125', 'cp1140', 'cp1250', 'cp1251', 'cp1252', 'cp1253', 'cp1254', - 'cp1255', 'cp1256', 'cp1257', 'cp1258', 'cp65001', 'euc-jp', 'euc-jis-2004', - 'euc-jisx0213', 'euc-kr', 'gb2312', 'gbk', 'gb18030', 'hz', 'iso2022-jp', - 'iso2022-jp-1', 'iso2022-jp-2', 'iso2022-jp-2004', 'iso2022-jp-3', - 'iso2022-jp-ext', 'iso2022-kr', 'iso8859-2', 'iso8859-3', 'iso8859-4', - 'iso8859-5', 'iso8859-6', 'iso8859-7', 'iso8859-8', 'iso8859-9', 'iso8859-10', - 'iso8859-11', 'iso8859-13', 'iso8859-14', 'iso8859-15', 'iso8859-16', 'johab', - 'koi8-r', 'koi8-t', 'koi8-u', 'kz1048', 'mac-cyrillic', 'mac-greek', - 'mac-iceland', 'mac-latin2', 'mac-roman', 'mac-turkish', 'ptcp154', - 'shift-jis', 'shift-jis-2004', 'shift-jisx0213', 'utf-32', 'utf-32-be', - 'utf-32-le', 'utf-16', 'utf-16-be', 'utf-16-le', 'utf-7', 'utf-8-sig'] - for e in encodings: - try: - fh = io.open(data, 'r', encoding=e) - fh.readlines() - fh.seek(0) - except UnicodeDecodeError: - print('got unicode error with %s , trying different encoding' % e) - else: - #print('opening the file with encoding: %s ' % e) - break - #print(os.path.basename(data)) - for lines in fh: - outfile = outfile+lines - fh.close() - outputfile = os.path.splitext(inputfile)[0]+'.inc' - outputfile = str(outputfile).replace(sourcedir, destdir) - #print(outputfile) - if not os.path.exists(os.path.dirname(outputfile)): - try: - os.makedirs(os.path.dirname(outputfile)) - except OSError as exc: # Guard against race condition - if exc.errno != errno.EEXIST: - raise - newfile = open(outputfile, "w") - newfile.write(outfile) - newfile.close() - -def single_thread(): - t1 = time.time() - cnt = sourcedir_filecnt(sourcedir) - print('Files in: '+str(cnt)) - process_files(sourcedir, destdir) - print('Single thread process time: '+str(time.time()-t1)) - -def multi_thread(): - global filelist - global exitFlag - - t1 = time.time() - cnt = sourcedir_filecnt(sourcedir) - threadList = ["thread 1", "thread 2", "thread 3", "thread 4", "thread 5"] - nameList = filelist - threadID = 0 - - # Create new threads - for tName in threadList: - thread = myThread(threadID, tName, workQueue) - thread.start() - threads.append(thread) - threadID += 1 - - # Fill the queue - queueLock.acquire() - for word in nameList: - workQueue.put(word) - queueLock.release() - - # Wait for queue to empty - while not workQueue.empty(): - pass - - # Notify threads it's time to exit - exitFlag = 1 - - # Wait for all threads to complete - for t in threads: - t.join() - print ("Exiting Main Thread") - print('Files in: '+str(cnt)) - print('Multi thread process time: '+str(time.time()-t1)) - -def async_process(num): - t1 = time.time() - cnt = sourcedir_filecnt(sourcedir) - - pool = mp.Pool(processes=num) - #[pool.apply_async(process_file, args=(f,)) for f in filelist] - pool.map(process_file, filelist) - #output = [p.get() for p in results] - #print(output) - - # Setup a list of processes that we want to run - #processes = [mp.Process(target=rand_string, args=(5, output)) for x in range(8)] - #processes = [mp.Process(target=process_files, args=(sourcedir, destdir)) for x in range(num)] - - # Run processes - #for p in processes: - #p.start() - - # Exit the completed processes - #for p in processes: - #p.join() - - # Get process results from the output queue - #results = [output.get() for p in processes] - - #print(results) - print('Files in: '+str(cnt)) - print('Multi ('+str(num)+') processes time: '+str(time.time()-t1)) - -def cleanup(dest): - shutil.rmtree(dest) - -#sourcedir = 'C:/Users/dksojlg/Documents/gtk+-3.22.26' -sourcedir = '/usr/include' -#destdir = 'C:/Users/dksojlg/Documents/include' -destdir = '/data_2/include' - -#print(cpu_info) -#single_thread() #2543 files - Single thread process time: 1.3732633590698242 sec. -#os.rmdir(destdir) -#multi_thread() #2543 files - Syncronized 5 threads process time: 80.25179100036621 sec. -#os.rmdir(destdir) -async_process(16) #2543 files - 2 processes, process time: 1.2379400730133057 sec. -#cleanup(destdir) -#async_process(4) #2543 files - 2 processes, process time: 2.6622860431671143 sec. -#cleanup(destdir) -#async_process(6) #2543 files - 2 processes, process time: 2.6622860431671143 sec. -#cleanup(destdir) -#async_process(8) #2543 files - 2 processes, process time: 2.6622860431671143 sec. -#cleanup(destdir) diff --git a/pool_test.py b/pool_test.py deleted file mode 100644 index d7db443..0000000 --- a/pool_test.py +++ /dev/null @@ -1,20 +0,0 @@ -from multiprocessing import Pool - -import time - -work = (["A", 5], ["B", 2], ["C", 1], ["D", 3]) - - -def work_log(work_data): - print(" Process %s waiting %s seconds" % (work_data[0], work_data[1])) - time.sleep(int(work_data[1])) - print(" Process %s Finished." % work_data[0]) - - -def pool_handler(): - p = Pool(2) - p.map(work_log, work) - - -if __name__ == '__main__': - pool_handler() \ No newline at end of file diff --git a/thread_priority_queue.py b/thread_priority_queue.py deleted file mode 100644 index 08daa57..0000000 --- a/thread_priority_queue.py +++ /dev/null @@ -1,59 +0,0 @@ -from Queue import Queue -import threading -import time - -exitFlag = 0 - -class myThread (threading.Thread): - def __init__(self, threadID, name, q): - threading.Thread.__init__(self) - self.threadID = threadID - self.name = name - self.q = q - def run(self): - print ("Starting " + self.name) - process_data(self.name, self.q) - print ("Exiting " + self.name) - -def process_data(threadName, q): - while not exitFlag: - queueLock.acquire() - if not workQueue.empty(): - data = q.get() - queueLock.release() - print ("%s processing %s" % (threadName, data)) - else: - queueLock.release() - time.sleep(1) - -threadList = ["Thread-1", "Thread-2", "Thread-3"] -nameList = ["One", "Two", "Three", "Four", "Five"] -queueLock = threading.Lock() -workQueue = Queue.Queue(10) -threads = [] -threadID = 1 - -# Create new threads -for tName in threadList: - thread = myThread(threadID, tName, workQueue) - thread.start() - threads.append(thread) - threadID += 1 - -# Fill the queue -queueLock.acquire() -for word in nameList: - workQueue.put(word) -queueLock.release() - -# Wait for queue to empty -while not workQueue.empty(): - pass - -# Notify threads it's time to exit -exitFlag = 1 - -# Wait for all threads to complete -for t in threads: - t.join() -print ("Exiting Main Thread") \ No newline at end of file From 141d015689489ab71147b84ed248aa1fc87803f7 Mon Sep 17 00:00:00 2001 From: Jan Lerking Date: Tue, 20 Feb 2018 19:06:59 +0100 Subject: [PATCH 15/37] update --- Multiprocessing test/calc_pi_linux.py | 7 +++++-- h2inc.py | 6 +++--- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/Multiprocessing test/calc_pi_linux.py b/Multiprocessing test/calc_pi_linux.py index b6ef69d..f27b52a 100644 --- a/Multiprocessing test/calc_pi_linux.py +++ b/Multiprocessing test/calc_pi_linux.py @@ -21,7 +21,9 @@ from multiprocessing import Queue, Process import queue from decimal import Decimal, getcontext -DELAY1 = 80 +import time + +DELAY1 = 20 DELAY2 = 20 class Example(Frame): @@ -116,7 +118,8 @@ class Example(Frame): print (self.p1.is_alive()) queue.put(pi) - print("end") + print("end") + time.sleep(1) def main(): diff --git a/h2inc.py b/h2inc.py index 1ac0511..c121503 100755 --- a/h2inc.py +++ b/h2inc.py @@ -53,13 +53,13 @@ def sourcedir_foldercnt(sourcedir): #print(len(folderlist)) return cnt -def process_files(source, dest): +def process_files(gui, source, dest): global sourcedir global destdir sourcedir = source destdir = dest pool = mp.Pool(processes=num_cores) - pool.map(process_file, filelist) + pool.map(process_file, filelist, gui) def process_file(data): outfile = '' @@ -106,7 +106,7 @@ def process_file(data): newfile = open(outputfile, "w") newfile.write(outfile) newfile.close() - + def async_process(num): pool = mp.Pool(processes=num) pool.map(process_file, filelist) \ No newline at end of file From e78aad8c0abf89022369b22357ef96cc7ea1f6b6 Mon Sep 17 00:00:00 2001 From: Jan Lerking Date: Tue, 20 Feb 2018 21:00:31 +0100 Subject: [PATCH 16/37] Added some gtk testing --- Gui test/gtk_test.py | 48 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 Gui test/gtk_test.py diff --git a/Gui test/gtk_test.py b/Gui test/gtk_test.py new file mode 100644 index 0000000..b7b0d0f --- /dev/null +++ b/Gui test/gtk_test.py @@ -0,0 +1,48 @@ +import gi +gi.require_version('Gtk', '3.0') +from gi.repository import Gtk +import time + +class MyWindow(Gtk.Window): + + def __init__(self): + Gtk.Window.__init__(self, title = 'Hello World') + + self.val = 0 + + self.box = Gtk.Box(spacing=6) + self.add(self.box) + + self.button1 = Gtk.Button(label = 'Hello') + self.button1.connect('clicked', self.on_button1_clicked) + self.box.pack_start(self.button1, True, True, 0) + + self.button2 = Gtk.Button(label = 'Goodbye') + self.button2.connect('clicked', self.on_button2_clicked) + self.box.pack_start(self.button2, True, True, 0) + + self.pbar = Gtk.ProgressBar() + self.pbar.set_fraction(0.0) + self.box.pack_start(self.pbar, True, True, 0) + + def on_button1_clicked(self, widget): + print('Hello') + while self.val < 1: + self.update_pbar(self.pbar) + print('Goodbye') + Gtk.main_quit() + + def on_button2_clicked(self, widget): + print('Goodbye') + + def update_pbar(self, widget): + self.val += 0.1 + self.pbar.set_fraction(self.val) + time.sleep(0.5) + while Gtk.events_pending(): + Gtk.main_iteration() + +win = MyWindow() +win.connect('delete-event', Gtk.main_quit) +win.show_all() +Gtk.main() From d7b4b3654ef7fa09040139b34d09b14e98464e4b Mon Sep 17 00:00:00 2001 From: Lerking <33354709+Lerking@users.noreply.github.com> Date: Wed, 21 Feb 2018 08:12:02 +0100 Subject: [PATCH 17/37] updated --- __pycache__/h2inc.cpython-36.pyc | Bin 0 -> 3518 bytes __pycache__/h2inc_gui.cpython-36.pyc | Bin 0 -> 5631 bytes __pycache__/h2inc_parser.cpython-36.pyc | Bin 0 -> 3949 bytes h2inc.py | 2 +- h2inc_gui.py | 2 +- old_h2inc/h2inc_gui_old.py | 173 ++++++++++++++++++++++++ old_h2inc/h2inc_old.py | 112 +++++++++++++++ 7 files changed, 287 insertions(+), 2 deletions(-) create mode 100644 __pycache__/h2inc.cpython-36.pyc create mode 100644 __pycache__/h2inc_gui.cpython-36.pyc create mode 100644 __pycache__/h2inc_parser.cpython-36.pyc create mode 100644 old_h2inc/h2inc_gui_old.py create mode 100644 old_h2inc/h2inc_old.py diff --git a/__pycache__/h2inc.cpython-36.pyc b/__pycache__/h2inc.cpython-36.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f4763d652483a29ac1e928d27ba7aeb698c1c56f GIT binary patch literal 3518 zcmZu!OLH5?5uVvy01FTVAEHRf@dH+DDM$7K`j>hf~tKeu1E z{iUH`4_B!0q?U`OnHlI*=U)XS}hBy(N7GLt>hrpO7U3FZrN?tN}sY; zrp=rsELmXAGMG*d%Zg`qS1(y>4J=4}#aio-gcQ;nU_%<|24r9f=}pMOGSW@pPgo)M z4bFz`7YvmiIYN4Uf9R8KYtOjE|1tH-($}Yj?Jkt zHK*43USy&CV48>$UE`8{+I)vOR+yN#TyoA{TeO!wSRbnLk zVTjsk8H|PNfiINl4nvhW4f^1TeP`gQj2xfDqErboMaM+`&cqEGuH=4v{8V~E9z#zC z-G2LcFL1{LZy3tsPQ@R(#epM3PsGCZ=t8YskM@!r;ec&DZ%Y zL)^r+H-cGl(w!YP_=0 zlv6eehDf8T$Y6zg$k;B4b&@G+akYWmx0wf&3pW^y9^OY;(pX70xwjA`oeT`q_Je-l zI(_*NnJXJXyRMKBDI~(iO5ONBmLf+zv$T?JAt{#0VudP-yjUZR^mVdTBJ>8M)9=hy z+(NB;SR^flS)1I#Y-KTD7PpO@sry}%`tU~~BVOn017srZu*h(DXml~+z+ZFBmf1~& z$uM36Mk+fq z(%VWb;eZ88neDN!l5tFwMNX78LhJ9WOf4@O%O)C23MVryTw9h4TJ`mYpqv60y|Vw} zv7_T2v8e$Dm~_d|1w)q$T`Y8|Kn_-571m%KZomfIgiY9jTW}lRg73lG@D98S??E2k zhdc0n_y9hHyKoPF00sCV{0KgRB5cEbC_x!2P=y-Qp#eM4gkAU;ehd%bC-4w{3XkAt z@N@VC_TVx60)7df!e{U+_%-|nehd5XIUK<6;P>zZzJRCjB^<&LJcAbe0gmB0ynvVR zM{wW-TmbOeFg4GQ=%hS9m|7Qi<8ub5kp=D7=GE23@sh&pt3&7OkPYuxEVO!=uerF( z8GT^_{K(qG!@=1z_DUVG1FR3(`DT{4C3n~v~W zgwAaSUDQa0nJ9J1yGvmp3#YG5SFlcYCxi`~&AtEn%Gl~Sp8EuEJts|$Q>fv+#{nB#8HaZV=5 zUf?&2Vn)Lm9jjz-QZCh+lkC88i|&Q+`+eV?q)BQE&+AQ=i1l5s?+js*AweGkTrKe; z7&yaNCmf5OFFPtV3URa4apAJ#pN7S*FDJQcHqPT*o@J8GR~3E6w0PoOvwd%pq}DRV zET-4JimEi}<3P#>grG@7JVMPQ=5@V#+pT>^&JaYe- zyo)T}8+fw%0U{@Qy75hN2%E`K-w$yUDN}|*rGz)?JFXYM)$fpA(mTiJTjF!*iy`4N zd*Jj4xna}iPqcHDz!eMwm3;j8=(8tZsKl8-pM?y*eopA9RDiKS+@;d~aCF69ElDcw zDdV(*uu-odgUVM*rg!eD)X+O4>+d|D;to#k5f*uX6~iZ!fI$wwRs9iXYp9UL$25cA zDi#B4TkEWwNh5}R8+;WZpusm00Ot6H*45sxA74U9*RbMk6lRYT!3jcSMu_0z4LBi4 z3qcY=H}M`-?^8u{j$wkKOvDo6E;YP~&(yLbFNW^R89ErEhNMfM5hj=*Eo{XJH06a< zD}!-A^k=%XyYz@JP)%hu7=Ee$IS4^(5VQ!Dj@t>Pq{1<#AOB>eMwoa6KzefGHt7<8 zTi6C^cQi)mGaiN_Pd$W_HTcpvC0-fLZecZgT?9Fm{a_HlxbHnA7$XUo*_a0UFdYV^ PZvNSZy=<@9w!Qs7{Y;!d literal 0 HcmV?d00001 diff --git a/__pycache__/h2inc_gui.cpython-36.pyc b/__pycache__/h2inc_gui.cpython-36.pyc new file mode 100644 index 0000000000000000000000000000000000000000..fbc111201b13e15ae220931c8e2f7d32acb6d7b5 GIT binary patch literal 5631 zcmbVQTW=f372eq;m&>atiMrXAO-a7diDaiOdNEQriKQ5B6Q!t}+tdI-aaIy3E?1o$ zCKf@rKop=s0rdwI0Sff3|Dlil33({ywNC>23;NXW%q}mI?4l@%J#%K}oVlF)?)Qtu z-2Boh|Jx1A`lmJVbAZ2tC;AtFumtN^0pl#Nxg9v%2~s>2q6xV2V$f@ifNEyc|sPY14N*Gr=sMHF%~o!)LIE z*Qo?^d@dQA=kplLbQbsm+H7YrSmH~;6@DeS%C9o(i6wF(|FtFZCpN$4vlELi`^)X? zCl<5N-Vg;*II)3W^OsK;uVQS4=0s-4suj<0W2=_d_OnCnD0QgQA2fFTj&|ScDHV2g z&UD|Gjli!tI=9#A_@dS5g!?+Z)m8FX=N}*V&BI+o4T{tg*%CUtqhzbQ|4BpY>}J^8 z?f5`zDb2Pu+tO@DvrlWb_BNmHy!+k9j9pH{1r6^Ise_x}q2P z66;P!VNW)(x2)3{nq8%5Cx*g~@Ffivk7U^NqsUN2O}gFiX9IZ$PxLl`Z*e9pZVM*t zuPyEfN2JiEL|V9L(;_20w64gC9NLV?ivn6t6d|>e_71++>NcNz{Ip4{(vR9K92v~A!n;RE6- zV5HD4icB(FXqSYCfA$o5R#O8;7I8{zmO)u=Pve$M6x58GU9eOI&)k9~iujjKU9*1q z+zRu=C8m-!7SLOW*H9I)4zBVkd_gS|6&@z0&0b5umkgb!D#;ErrzucgAxbg(duxx0 zikSP_6Z5BZ*Q*%6YB(0uoY)wx9PhuF?0*eZ*Ffb7hW+mksm$oIx-OPZA(zoeJQ&{YjTo$bNIuGRLKdj(WwK__&?vbZj8oThQFRrFWE zy8x*Z_oSp($K1pPy}~OKtyPm7uVH2_-upFrTRF`by{%2^&2ZhEv;-OvtI4Xj(7W|N zt$J&6RYlq$UN2eaHt^e62R4=5&Det(9ZP1&l})(rVE&HT{UX<$u|$letcf+FjhhqN zxC`#P;4WO+^X>%q6sXq3tz^d5g?ghKPtcH8l zJrm`kht&s@qX&KivCoW_pM|Q?sUigKOT@s3)&Am+k6@{)aj2{!e4-c155ml_Mk6`Y8YC>I-lrQ5>eZSY*I6Bte+nq+RD;kg9qEq0p`fJ0Cvr(_Nx-C_&Z@k?Jn~hHN z2uMs_v*i`g>P#~X0_4s!N=p6W=2*oLA98G@px6H5#Q6U$i&7#BG9y@*&LoJreT~W6 zIDx_C9m4JcQ2LGUwEkH7WMQj`Ev+UdUTy7FTdFETm|HhgRsTgRQXASi_9Je0L#WkU z(;MoUXTt6mjZV+!UbwdxWBH*0!?0@^V+T9~gChfrZ3k80MFZ;3B&N~o?u9ZiUeV7D zSMpcRSqXByWvrwEWN0O5k$2n9i4$8(N@vy3ljJ(>coRUo4Iyyp|01PNKxU-Gyp|j5Xx3R?84o7UidJefSlN`+1$kp%uA<&Q21w5lE5tP6H_K=yl4fz0R3y| z%cXekD&aKn+VKsUCmK3=di)$5`i)S5)seG z?%1J9RvLLFBan+cmK1(?dE~P7xOmlBGF`(%L5a4N)1ZvU4W}+8)}63OlCI7|k2AJ7;R-i`p9ok~#FEK7hr%q+&NOxr71e>vfLs?E(cx$@oswaw-~z+KLH-ytk}L+UtE5gJ;YCkrXWv(n z0=xVvff)ji2>gt|djvKK{D{Cjfp-CFZfu_>u+WswC_gyDy4)ZzW}c1DjCZ>@jZjvN zbIBU8E}n?kEj9y_EwCI67$yrN9RJ;9w7)bmvWppfWNef%N5-bmA!9qSlwEg}-FD;= zOeIB_F*2wXeRpI!nYO30!cGh#cdnN|*DIXs6~{ee2+nqYVRM|d$Ju$T+>+6je2OE= zpA+~60i&G{3EL*{4uP>oNanFJ$d`;ph9SbFCL`OifZqhsS9l^)rv)LBfJb7r`wJJv znMB7X1!4M8=p|m`xqk;TaN2k)eo6jDZF*pv)`j1&LtB0eKe6E(X~jsyv7p=Vo)c7G zM&Or5qi>JrLF*d&gpygGTHOaJOJxpo@=wfqiFc)bWkAaD+Sx-#{^2W7d`@3;F!$Tc zXI9t#3~$!$9PNa6F(#b&gcK1}eqaq;%s5G9p4yfhxJ%!=pOh`H*Hs}_gHg!);E_)N zv}@{_2(?m6b}1QnLF14OtDXFl~?$*|4Pj{ZhT3x_M&5IqzXv=sFWnz4- z2yqmhBKf0uHifcw{1c|NNr@bv@`so}x#?W?U0pF{!!S?Pdq)Dr%b!6X(XtlnFOPPA z73HfBQUWI&(!%$R;bH9Ue(7;fN~CY|q5}Vkb0W_-<6%m>IAkX2UI;RJW`Zm^=3gd` zd3Aiu@p1{>b}dUemNZs-B#f~?meyil+Irq}4iGja>-$u$DxaFTQ+KPT{O0%M;X zJLV!LsOU8BCrH_!dlj$XO?!@{nfGg}h@*!H%lMW?;*<^na!OPBm`W;+9^{~=h>@hk zxXPolE-tM|o^gF;N-HXc;>d1H*;r^Mj5bW_pyO!o2rrCOI3qV4M?7tWW0FN2^T~wG z5lzI>Zmf2m#mj1|PcLjd{;r0%AP%2sbvt2r^ePG91t}KhBf3Gf3UCc}TY(*u1yc&3 QN~U6$erL_N%u7@MKQC=nSpWb4 literal 0 HcmV?d00001 diff --git a/__pycache__/h2inc_parser.cpython-36.pyc b/__pycache__/h2inc_parser.cpython-36.pyc new file mode 100644 index 0000000000000000000000000000000000000000..914c3fc68e57b066243555de5f1ca29400ded932 GIT binary patch literal 3949 zcmbVPOK)4p6~1#H zd*uf~0RiMiffhwubXD{RwEv;Ytc#+{yKOhzWF2VH^gDAYQY2`nEr~fZbLPwp=ljk% zGyE)_PW1O!|O`!Eagy7&{QQs(_WHhycErPX`1sgH1AE)f|sR5FGm-=DZ1!Q z(~>tsm%LfJ?9I^!-aK9L7UZ0qmkTl}Q!*_xa#ChxPEN^bIU{H3!#`!bML9=T~5i!&Rgs^AJA z2gO3UhvM%*JTs030fo0l$EX`8#-?#1j?LG%I_9SF<@IB$V+CSAraliu$67Rw?T)CM z9s5KGL$@q$y;%Agdf3exY-IK}+T)F~dZUiP`uJT zElzJ&KY#qlm&$)k`~G3Q@$B*DVfE-hHUt0h^G|Ed>WzbnA4nB-p0p|o#%rxNr-}7O z%@3-F2d#}s-j_Fs8SP8jqETzgwe`kfwbJl6&^WU!SfS&$8nqxyJty^KU%q*DsK^gv zl~zkOso;bTPxZria0Iia3~e@0HWm-X%fatf3~o5l-z?2PfsbwoTTF?BaPduuj8GZq z`jqgGLJ* zUMTJq?-xtuC*@x}EIzsS;J)tcmGADB!-cV9#nNV&JveFvHTD7c82e+G(rlmgE{~mg z|Ito)duO{;45x?2x3@~0#jT#hkbLTojAM`|%?wW}-P?JzS?q0$SZExP40A&^#hvY~ zaB@fm#i-3*F`OE*!3klO&?kc9*Pf1yZ*TF>9htMc`(SrO_edWrTIZ;VgN>Z);qKj^ z-oG1WqKyt(!5c;tiNHdDdC@##)o5rX^$aa&9-R7LKp3WtkV@ejL(3I2D5)PP8H`6I zVa9|iV3mG2au9Q9>u~%%8XS%t=7f`jxknwnshT|cZfk<{X{A+-xTHE{GgbR=iQ zP$0dH!CuW_?j02W3It_?Cfic^R$Kf9sVH|KK_D%!$ z_X^{|J$+vGPW0~JO3^4;KX>?LOi#>GYlqQj9^-1L8^qu*S==@J4;;o=bWEW@3BZ|-)=i=uq zkZ!bvt4W;DG$(pN@N0hL^@G6bg^`IMNRE;*06fCF5Ay(ec)tz@9UX(1um&+P7TFJh z#48m4GQ_)))p)Y)cszfDk^c@)cF7)wb?xTw!!WXVABJB8L(#_&0TKnr|BInDno@oU zzB~^8OBA1d?L2(v-BN?cmRtH#(cz_jgnpMZY?o7UH^w5)f|Gcc3G60VBw6r%)176( zIjfLqn>SZOvD!9QZ?@yhcm?1>Z@bIA7fU;~te*y`Yfmu%+Rn0U_)=Hs z#g_GRglnY^6>M9o^6a2;Dt@f^o!^e<&(_cP*U$IX&-d0x9j%W>+sQ}kqvG}Rz4i0G z_4CpCg^NfFZ}3k5F<)yw)6aJOScpnVm%F$(Gei;FC-vAD`&fdzZ9 z`hdkH7FSp-vshs<&0?Mf1F9J5h;V)73 0: + tempstr = 'Number of headers: '+str(filecnt) + temptot = 'Total progress: 0 of '+str(filecnt) + print ('Source directory: ', sourcedir.get()) + self.destlabel.config(state=NORMAL) + self.destentry.config(state=NORMAL) + self.destdir_button.config(state=NORMAL) + self.infofiles.config(text=tempstr) + self.totallabel.config(text=temptot) + self.filecnt = filecnt + foldercnt = sourcedir_foldercnt(root.directory) + if foldercnt > 0: + tempstr = 'Number of folders: '+str(foldercnt) + self.infofolders.config(text=tempstr) + + def select_destdir(self, destdir): + root.directory = filedialog.askdirectory() + if root.directory: + destdir.set(root.directory) + print ('Destination directory: ', destdir.get()) + self.incchkbox.config(state=NORMAL) + self.infofolders.config(state=NORMAL) + self.infofiles.config(state=NORMAL) + self.translate_button.config(state=NORMAL) + #self.cfilelabel.config(state=NORMAL) + self.totallabel.config(state=NORMAL) + + def translate(self, destdir, sourcedir, addinc): + doinc = addinc.get() + dest = destdir.get() + source = sourcedir.get() + if doinc == 'yes': + dest = dest+'/include' + print(os.path.exists(os.path.dirname(dest))) + if not os.path.exists(os.path.dirname(dest)): + try: + os.makedirs(os.path.dirname(dest)) + except OSError as exc: # Guard against race condition + if exc.errno != errno.EEXIST: + raise + destdir.set(dest) + print ('Destination directory: ', destdir.get()) + process_files(self, source, dest) + + def cfileprogress_update(cnt): + self.cfilevar = cnt + + def currentfile_update(current): + self.currentfile = 'Current file: '+current + +root = Tk() +root.update() +#root.minsize(350, 210) +#width = (root.winfo_screenwidth()/2)-(350/2) +#height = (root.winfo_screenheight()/2)-(210/2) +#root.geometry('+%d+%d' % (width, height)) +root.resizable(False, False) +h2incgui = h2incGUI(root) +root.mainloop() \ No newline at end of file diff --git a/old_h2inc/h2inc_old.py b/old_h2inc/h2inc_old.py new file mode 100644 index 0000000..233a495 --- /dev/null +++ b/old_h2inc/h2inc_old.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python3.5 + +# You are free to use and/or change this code for +# your own needs. + +# Original code (c)2018 Jan Lerking +# Program to convert C-header (*.h) files to nasm include files (*.inc), +# for direct usage in assembly programming using nasm/yasm. + +import os +import sys +import multiprocessing as mp +import io +from h2inc_parser import parseline, parseparsed +import h2inc_gui_old + +tupline = [] +preproc = () +filelist = [] +folderlist = [] +cnt = 0 +sourcedir = '' +destdir = '' +num_cores = mp.cpu_count() + +def sourcedir_filecnt(sourcedir): + ### Return the number of files, ending with '.h', in sourcedir - including subdirectories ### + cnt = 0 + global filelist + for folderName, subfolders, files in os.walk(sourcedir): + for file in files: + if file.lower().endswith('.h'): + cnt += 1 + filelist += [folderName+'/'+file] + print(folderName+'/'+file) + #print(filelist) + return cnt + +def sourcedir_foldercnt(sourcedir): + ### Return the number of folders, if it contains '*.h' files, in sourcedir - including subdirectories ### + global cnt + global folderlist + for folderName, subfolders, files in os.walk(sourcedir): + if subfolders: + for subfolder in subfolders: + sourcedir_foldercnt(subfolder) + tempf = [file for file in files if file.lower().endswith('.h')] + if tempf: + cnt = cnt+1 + #print(folderName) + folderlist += [folderName] + #print(folderlist) + #print(len(folderlist)) + return cnt + +def process_files(gui, source, dest): + global sourcedir + global destdir + sourcedir = source + destdir = dest + pool = mp.Pool(processes=num_cores) + pool.map(process_file, filelist) + +def process_file(data): + outfile = '' + inputfile = data + encodings = ['utf-8', 'latin-1', 'windows-1250', 'windows-1252', 'ascii', + 'big5', 'big5hkscs', 'cp037', 'cp273', 'cp424', 'cp437', 'cp500', + 'cp720', 'cp737', 'cp775', 'cp850', 'cp852', 'cp855', 'cp856', + 'cp857', 'cp858', 'cp860', 'cp861', 'cp862', 'cp863', 'cp864', 'cp865', + 'cp866', 'cp869', 'cp874', 'cp875', 'cp932', 'cp949', 'cp950', 'cp1006', + 'cp1026', 'cp1125', 'cp1140', 'cp1250', 'cp1251', 'cp1252', 'cp1253', 'cp1254', + 'cp1255', 'cp1256', 'cp1257', 'cp1258', 'cp65001', 'euc-jp', 'euc-jis-2004', + 'euc-jisx0213', 'euc-kr', 'gb2312', 'gbk', 'gb18030', 'hz', 'iso2022-jp', + 'iso2022-jp-1', 'iso2022-jp-2', 'iso2022-jp-2004', 'iso2022-jp-3', + 'iso2022-jp-ext', 'iso2022-kr', 'iso8859-2', 'iso8859-3', 'iso8859-4', + 'iso8859-5', 'iso8859-6', 'iso8859-7', 'iso8859-8', 'iso8859-9', 'iso8859-10', + 'iso8859-11', 'iso8859-13', 'iso8859-14', 'iso8859-15', 'iso8859-16', 'johab', + 'koi8-r', 'koi8-t', 'koi8-u', 'kz1048', 'mac-cyrillic', 'mac-greek', + 'mac-iceland', 'mac-latin2', 'mac-roman', 'mac-turkish', 'ptcp154', + 'shift-jis', 'shift-jis-2004', 'shift-jisx0213', 'utf-32', 'utf-32-be', + 'utf-32-le', 'utf-16', 'utf-16-be', 'utf-16-le', 'utf-7', 'utf-8-sig'] + for e in encodings: + try: + fh = io.open(data, 'r', encoding=e) + fh.readlines() + fh.seek(0) + except UnicodeDecodeError: + print('got unicode error with %s , trying different encoding' % e) + else: + #print('opening the file with encoding: %s ' % e) + break + #print(os.path.basename(data)) + for lines in fh: + outfile = outfile+lines + fh.close() + outputfile = os.path.splitext(inputfile)[0]+'.inc' + outputfile = str(outputfile).replace(sourcedir, destdir) + #print(outputfile) + if not os.path.exists(os.path.dirname(outputfile)): + try: + os.makedirs(os.path.dirname(outputfile)) + except OSError as exc: # Guard against race condition + if exc.errno != errno.EEXIST: + raise + newfile = open(outputfile, "w") + newfile.write(outfile) + newfile.close() + +def async_process(num): + pool = mp.Pool(processes=num) + pool.map(process_file, filelist) \ No newline at end of file From 0a3a0105fe2230aa8309fa7705279a8ee830b55b Mon Sep 17 00:00:00 2001 From: Lerking <33354709+Lerking@users.noreply.github.com> Date: Wed, 21 Feb 2018 14:02:02 +0100 Subject: [PATCH 18/37] Add files via upload --- Gui test/gtk_test_1.py | 112 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 112 insertions(+) create mode 100644 Gui test/gtk_test_1.py diff --git a/Gui test/gtk_test_1.py b/Gui test/gtk_test_1.py new file mode 100644 index 0000000..53f3dcd --- /dev/null +++ b/Gui test/gtk_test_1.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python3.5 + +# You are free to use and/or change this code for +# your own needs. + +# Original code (c)2018 Jan Lerking +# Program to convert C-header (*.h) files to nasm include files (*.inc), +# for direct usage in assembly programming using nasm/yasm. + +import gi +gi.require_version('Gtk', '3.0') +from gi.repository import Gtk + +class h2incGUI(Gtk.Window): + def __init__(self): + Gtk.Window.__init__(self) + self.set_title('Translate C-header files to Nasm include files!') + self.set_default_size(200, 200) + self.set_border_width(5) + self.connect("destroy", Gtk.main_quit) + + self.sourcedir = 'Select source directory!' + self.destdir = 'Select destination directory!' + self.addinc = False + self.totalvar = 0 + self.filecnt = 0 + self.infofolder = 'Number of folders: 0' + self.infofile = 'Number of headers: 0' + self.currentfile = 'Current file: ' + self.totprogress = 'Total progress: ' + self.val = 0 + + self.maingrid = Gtk.Grid() + self.add(self.maingrid) + + self.frame = Gtk.Frame(label='Select folders') + self.maingrid.add(self.frame) + + self.framegrid = Gtk.Grid() + self.frame.add(self.framegrid) + + self.sourcelabel = Gtk.Label(label='Source: ') + self.framegrid.attach(self.sourcelabel, 1, 1, 1, 1) + + self.sourceentry = Gtk.Entry() + self.sourceentry.set_text(self.sourcedir) + self.framegrid.attach(self.sourceentry, 2, 1, 1, 1) + + self.sourcedir_button = Gtk.Button(label="Source directory...") + #self.sourcedir_button.connect('clicked', self.select_sourcedir(self.sourcedir)) + self.framegrid.attach(self.sourcedir_button, 3, 1, 1, 1) + + self.destbox = Gtk.Box(orientation=Gtk.Orientation.HORIZONTAL, spacing=6) + self.framegrid.attach(self.destbox, 1, 2, 1, 1) + + self.destlabel = Gtk.Label(label='Destination: ') + self.framegrid.attach(self.destlabel, 2, 2, 1, 1) + + self.destentry = Gtk.Entry() + self.destentry.set_text(self.destdir) + #self.destbox.add(self.destentry) + + self.destdir_button = Gtk.Button(label="Destination directory...") + #self.destdir_button.connect('clicked', self.select_destdir(self.destdir)) + #self.destbox.add(self.destdir_button) + + self.incchkbox = Gtk.CheckButton('Create "include" folder if it does not exist.') + self.incchkbox.set_active(False) + #self.incchkbox.connect("toggled", self.on_button_toggled, self.addinc) + #self.framebox.add(self.incchkbox) + + self.transframe = Gtk.Frame(label='Translation') + #self.mainbox.add(self.transframe) + + self.framebox_1 = Gtk.Box(orientation=Gtk.Orientation.VERTICAL, spacing=6) + #self.transframe.add(self.framebox_1) + + self.infoframe = Gtk.Frame(label='Source information') + #self.framebox_1.add(self.infoframe) + + self.infobox = Gtk.Box(orientation=Gtk.Orientation.VERTICAL, spacing=6) + #self.infoframe.add(self.infobox) + + self.infofolders = Gtk.Label(label=self.infofolder) + #self.infobox.add(self.infofolders) + + self.infofiles = Gtk.Label(label=self.infofile) + #self.infobox.add(self.infofiles) + + self.translate_button = Gtk.Button(label="Translate!") + #self.translate_button.connect('clicked', self.translate(self.destdir, self.sourcedir, self.addinc)) + #self.framebox_1.add(self.translate_button) + + self.progressframe = Gtk.Frame(label='Progress') + #self.framebox_1.add(self.progressframe) + + self.progressbox = Gtk.Box(orientation=Gtk.Orientation.VERTICAL, spacing=6) + #self.progressframe.add(self.progressbox) + + self.cfilelabel = Gtk.Label(label=self.currentfile) + #self.progressbox.add(self.cfilelabel) + + self.totallabel = Gtk.Label(label=self.totprogress) + #self.progressbox.add(self.totallabel) + + self.totalprogress = Gtk.ProgressBar() + self.totalprogress.set_fraction(0.0) + #self.progressbox.add(self.totalprogress) + +win = h2incGUI() +win.show_all() +Gtk.main() \ No newline at end of file From 0b63d4edc4cb83ef8c0601daf64a8c4ff52c9bae Mon Sep 17 00:00:00 2001 From: Jan Lerking Date: Wed, 21 Feb 2018 18:51:23 +0100 Subject: [PATCH 19/37] Added Glade gui builder file --- Gui test/gtk_test_1.py | 21 +- __pycache__/h2inc.cpython-35.pyc | Bin 3818 -> 3823 bytes __pycache__/h2inc_gui.cpython-35.pyc | Bin 6564 -> 6502 bytes h2inc.glade | 393 +++++++++++++++++++++++++++ 4 files changed, 406 insertions(+), 8 deletions(-) create mode 100644 h2inc.glade diff --git a/Gui test/gtk_test_1.py b/Gui test/gtk_test_1.py index 53f3dcd..517805f 100644 --- a/Gui test/gtk_test_1.py +++ b/Gui test/gtk_test_1.py @@ -16,7 +16,7 @@ class h2incGUI(Gtk.Window): Gtk.Window.__init__(self) self.set_title('Translate C-header files to Nasm include files!') self.set_default_size(200, 200) - self.set_border_width(5) + self.set_border_width(10) self.connect("destroy", Gtk.main_quit) self.sourcedir = 'Select source directory!' @@ -37,32 +37,37 @@ class h2incGUI(Gtk.Window): self.maingrid.add(self.frame) self.framegrid = Gtk.Grid() + self.framegrid.set_border_width(5) self.frame.add(self.framegrid) + self.sourcebox = Gtk.Box(orientation=Gtk.Orientation.HORIZONTAL, spacing=6) + self.framegrid.attach(self.sourcebox, 0, 0, 5, 1) + self.sourcelabel = Gtk.Label(label='Source: ') - self.framegrid.attach(self.sourcelabel, 1, 1, 1, 1) + self.sourcelabel.set_justify(Gtk.Justification.RIGHT) + self.sourcebox.add(self.sourcelabel) self.sourceentry = Gtk.Entry() self.sourceentry.set_text(self.sourcedir) - self.framegrid.attach(self.sourceentry, 2, 1, 1, 1) + self.sourcebox.add(self.sourceentry) self.sourcedir_button = Gtk.Button(label="Source directory...") #self.sourcedir_button.connect('clicked', self.select_sourcedir(self.sourcedir)) - self.framegrid.attach(self.sourcedir_button, 3, 1, 1, 1) + self.sourcebox.add(self.sourcedir_button) self.destbox = Gtk.Box(orientation=Gtk.Orientation.HORIZONTAL, spacing=6) - self.framegrid.attach(self.destbox, 1, 2, 1, 1) + self.framegrid.attach(self.destbox, 0, 1, 5, 1) self.destlabel = Gtk.Label(label='Destination: ') - self.framegrid.attach(self.destlabel, 2, 2, 1, 1) + self.destbox.add(self.destlabel) self.destentry = Gtk.Entry() self.destentry.set_text(self.destdir) - #self.destbox.add(self.destentry) + self.destbox.add(self.destentry) self.destdir_button = Gtk.Button(label="Destination directory...") #self.destdir_button.connect('clicked', self.select_destdir(self.destdir)) - #self.destbox.add(self.destdir_button) + self.destbox.add(self.destdir_button) self.incchkbox = Gtk.CheckButton('Create "include" folder if it does not exist.') self.incchkbox.set_active(False) diff --git a/__pycache__/h2inc.cpython-35.pyc b/__pycache__/h2inc.cpython-35.pyc index 2dec8fc417e5ee7a12f514df8a54c322b679b88d..92eab8b66bf904bb493a8665341971043a080e17 100644 GIT binary patch delta 136 zcmaDQ`(Bn)jF*>-b$V~qvW=Xq?ncgF7E2}u2w(kFPsJ7XTbqWWg>g3D3`izy6nfcThV<&6z<#4M3 P%_&j>5tWk{^7#P(%03v+ diff --git a/__pycache__/h2inc_gui.cpython-35.pyc b/__pycache__/h2inc_gui.cpython-35.pyc index 2867d0063d7d36da02b5e6ec7c7b8e042b105fa5..b5fb497bede08f2cd1cadfa5df0cd08adf0d4543 100644 GIT binary patch delta 2882 zcmb7GOK)3M5T4`rD~U;*IIrgIBuAV}y(Gvm919S%5AZ(1`y&B(R7er-rC_fQ zWVG#w`+kryka5CeA%r;qcta*YCTUMs(|k1zdyjxjfkZ2EAmYsduw)v{K`@6PYi95> z3|WbOgCK_pk0HVl;q7jrkrpj!Wjw7cA7;;Jp-2XZc(Y9$7Un42voNbP$YH)GJeo1y zj&;&tnoPqnwxSCObKDol2*@ma2(W|cje;D54*)3xRD}t4o9Lt*ck>_S&@T8x^Jh2V zg*C|>lO6{KTvxXS7rPQb-wvT!F9Ce?X^<{q5acY#Igs-p7qC_#s`ngw z&UsoW((XFHpw-=Jc*Czq*#>zW=8zeD{hjk(WuBGhJteH#A}7_>8{h)?i{!V3C%6O_}#Mr3Hm$7Fza% z@U_}g)msQ%JA?+9z!h5Y!q4+z-eW!gkgmY~d-!*s`x?m6`!z5T_caop=n9Ljc%rNS znaYaQmCJe9TWv)y*Fdg=d=%tk$j@J5YiwHc_#gN9F}{Cma{o6uUdH0NgBwA^22=AjsL})HW)7mHOPu)4?o)6Ddk* zR7>pF;|{vKfu9C`K#RSqYP(8rG)_XeJ&Dm-q^1nLYHEqwxHzm;&0^UJRccb#l2SKy zRWYhXQ$LB5Z?sZ8Fe=ST-U_HGJWyi@`G7h{YdXMn@AGw)Bp-@caqMffN+CG0(vDHq z-zh3BX4X(0S5B0UN)(k*-Kp)=%2G!&uhF0-?Hm2mSczu;EaLtCFA%o}c5iA)FTq<# z*T;1AR>~L}*K>=6{RO7lInASwC-7Y$od`sh(4@xg!NJ$9 z3$&yQPOxDrCtO!X)pUZpx~ZNd(^CY41eXZ537#dmOt48XLhuYiKH`SQ6(W-pHT6mz zc{N^OE?V3z1lENUNcnnW#{P4#C_b}a8yXXz+WSMtw#dv{6l=a;MM>lZf)@#1BH-9x zrqwlqrwLBurHI^6pQF`f1h0Pldfb$!ow~Atr5wO3{EGEcs zl{{khADO?gW=9b?!vEfa;7(BJ6}4hN7|y3T8jkA^2Q8Rw)|Qa;;WBD^wl3_zz7( BBhml> delta 2904 zcma)8OK%%h6h7m`Z^wDsPU4ihO`60>>$FW<5EO(|l?~8BQ$cyvaebZCsUPX}1hrM9 z5etI3t$UURTlh&{sN>95*s8?koW_z;X7wM&e(NDWzF%q_n!NG=Y03vvHv*z z{c1ElG!%PWdGO&+52H^4#=iY~VBJzddO%c+v)wXiD?utmdj=Ui4#+qx<6T1nha_-_ zwvwbGoj{C-NkvJ;*zOsIVU7@Xs5q$vkC|gswvN))7^xvrSWS|}U)VZAYLv`zGAAfw zrtqXGqYyVvYJ%+^g(M^F?j_2!VVNuCaZSqPWEU4xfh0u?e8vT~fE+Z;DGwtpj^V*f z%W%3I24xD2 zW@P5CZpKUNkYF70(n3Oj@K<-v2f+u4&=*J@BJ(in#o=`7wcLUO9&%2QbKS`O3uasd zcH#Opz(PlZ!`&Jj5tbuhv7{B;0{j}h*i!&vyNHeo!l*P@j+6^HK(k^` zPnUsreNkmwQ2m z3mz$&vD1>h`Ua&&ghF5IpeNlK|(au~INUe}MN$M2vpQaLCAY}T6oBbyrY|&y3%Ux4`R3<;ki&# zS5br$y9wuBvtgEM&S1T%Do$Kg%!;m88zr-{ALZC^y>zEq->K)VfIf-|Jq42w=)>&u z1pNN}|GoUxYN&#;$(aUx~b@6(cYB4!ip?1g!%%vn_%8_kVoO;xZgJ{FvKqU=h~ zVUAr$<{-=E6mj3+Ec|M-&`zXStTw7%s4D>!ar@z*Ddw@Efu}VGLq^vAW$a4!AzTwT2Y(je z1gJQ`7C#dD2m8YKUJxa1CGDTbS1e9HKdi&}(jT$;n9V0_F0v`IS!Z*Z%|Tt9xm(g2 zJTyEXMXhwBQj;1zf#s|v+k=z~*l-_DYl2t=V;GW-7~{T=-I$nsqo0HTt80RCdww>#ux$hkNe0qAMC#zL^ zBYkT|y5cq7X*dHta@&8WKhH@3r-guUy9@niVt<|4EWD3(%_pO|R=vum$>uhj17cSb zyD)hSJ$rTXx^c~ZB>t1h?0G&3C$zJzaQ&PpZf*5m=?2<8a`QCq`{qv7iPTHgMy=W0 Lej4)R)Vcov(7rX8 diff --git a/h2inc.glade b/h2inc.glade new file mode 100644 index 0000000..4702fd7 --- /dev/null +++ b/h2inc.glade @@ -0,0 +1,393 @@ + + + + + + False + normal + select-folder + + + False + vertical + 2 + + + False + end + + + + + + + + + False + False + 0 + + + + + + + + + + False + 5 + 5 + 5 + 5 + + + True + False + + + True + False + 0 + etched-out + + + True + False + 12 + + + True + False + + + True + False + end + 5 + 5 + 5 + True + Source: + right + + + 0 + 0 + 2 + + + + + True + True + 5 + 5 + 5 + 5 + Select source directory! + + + 2 + 0 + 2 + + + + + Source directory... + True + True + True + start + 5 + 5 + 5 + 5 + + + 4 + 0 + 2 + + + + + True + False + False + end + 5 + 5 + 5 + True + Destination: + right + start + 2.2351741811588166e-10 + + + 0 + 1 + 2 + + + + + True + False + True + 5 + 5 + 5 + 5 + Select destination directory! + + + 2 + 1 + 2 + + + + + Destination directory... + True + False + True + True + start + 5 + 5 + 5 + 5 + + + 4 + 1 + 2 + + + + + Create "include" folder if it does not exist. + True + False + True + False + 5 + 5 + 0 + True + + + 0 + 2 + 6 + + + + + + + + + True + False + Select folders + + + + + 0 + 0 + 5 + + + + + True + False + False + 5 + 5 + 0 + etched-out + + + True + False + 12 + + + True + False + + + True + False + 5 + 5 + True + 0 + etched-out + + + True + False + 12 + + + True + False + vertical + + + True + False + start + 5 + 5 + True + Number of folders: 0 + + + False + True + 0 + + + + + True + False + start + 5 + 5 + Number of files: 0 + + + False + True + 1 + + + + + + + + + True + False + Source information + + + + + 0 + 0 + + + + + Translate! + True + True + True + start + 5 + 5 + + + 0 + 1 + + + + + True + False + 5 + 5 + 5 + 0 + etched-out + + + True + False + 12 + + + True + False + vertical + + + True + False + start + 5 + 5 + Total progress: + + + False + True + 0 + + + + + True + False + 5 + 5 + 5 + + + False + True + 1 + + + + + + + + + True + False + Progress + + + + + 0 + 2 + + + + + + + + + True + False + Translation + + + + + 0 + 1 + 5 + + + + + + From cea9b90d6c9f87137e86e4f94c1797decf7bb4d9 Mon Sep 17 00:00:00 2001 From: Jan Lerking Date: Wed, 21 Feb 2018 20:55:45 +0100 Subject: [PATCH 20/37] updated --- Gui test/gtk3_glade_test.py | 21 ++ Gui test/h2inc (copy).xml | 393 ++++++++++++++++++++++++++++++++++++ Gui test/h2inc.glade | 367 +++++++++++++++++++++++++++++++++ Gui test/h2inc.glade~ | 367 +++++++++++++++++++++++++++++++++ h2inc.glade | 38 +--- h2inc.glade~ | 365 +++++++++++++++++++++++++++++++++ 6 files changed, 1519 insertions(+), 32 deletions(-) create mode 100644 Gui test/gtk3_glade_test.py create mode 100644 Gui test/h2inc (copy).xml create mode 100644 Gui test/h2inc.glade create mode 100644 Gui test/h2inc.glade~ create mode 100644 h2inc.glade~ diff --git a/Gui test/gtk3_glade_test.py b/Gui test/gtk3_glade_test.py new file mode 100644 index 0000000..fad456f --- /dev/null +++ b/Gui test/gtk3_glade_test.py @@ -0,0 +1,21 @@ +import gi +gi.require_version('Gtk', '3.0') +from gi.repository import Gtk + +class Handler: + def onDeleteWindow(self, *args): + Gtk.main_quit(*args) + + def onButtonPressed(self, button): + print("Hello World!") + +builder = Gtk.Builder() +builder.add_from_file("h2inc.glade") +builder.connect_signals(Handler()) + +window = builder.get_object("window1") +window.connect('delete-event', Gtk.main_quit) +window.show_all() + +Gtk.main() + diff --git a/Gui test/h2inc (copy).xml b/Gui test/h2inc (copy).xml new file mode 100644 index 0000000..4702fd7 --- /dev/null +++ b/Gui test/h2inc (copy).xml @@ -0,0 +1,393 @@ + + + + + + False + normal + select-folder + + + False + vertical + 2 + + + False + end + + + + + + + + + False + False + 0 + + + + + + + + + + False + 5 + 5 + 5 + 5 + + + True + False + + + True + False + 0 + etched-out + + + True + False + 12 + + + True + False + + + True + False + end + 5 + 5 + 5 + True + Source: + right + + + 0 + 0 + 2 + + + + + True + True + 5 + 5 + 5 + 5 + Select source directory! + + + 2 + 0 + 2 + + + + + Source directory... + True + True + True + start + 5 + 5 + 5 + 5 + + + 4 + 0 + 2 + + + + + True + False + False + end + 5 + 5 + 5 + True + Destination: + right + start + 2.2351741811588166e-10 + + + 0 + 1 + 2 + + + + + True + False + True + 5 + 5 + 5 + 5 + Select destination directory! + + + 2 + 1 + 2 + + + + + Destination directory... + True + False + True + True + start + 5 + 5 + 5 + 5 + + + 4 + 1 + 2 + + + + + Create "include" folder if it does not exist. + True + False + True + False + 5 + 5 + 0 + True + + + 0 + 2 + 6 + + + + + + + + + True + False + Select folders + + + + + 0 + 0 + 5 + + + + + True + False + False + 5 + 5 + 0 + etched-out + + + True + False + 12 + + + True + False + + + True + False + 5 + 5 + True + 0 + etched-out + + + True + False + 12 + + + True + False + vertical + + + True + False + start + 5 + 5 + True + Number of folders: 0 + + + False + True + 0 + + + + + True + False + start + 5 + 5 + Number of files: 0 + + + False + True + 1 + + + + + + + + + True + False + Source information + + + + + 0 + 0 + + + + + Translate! + True + True + True + start + 5 + 5 + + + 0 + 1 + + + + + True + False + 5 + 5 + 5 + 0 + etched-out + + + True + False + 12 + + + True + False + vertical + + + True + False + start + 5 + 5 + Total progress: + + + False + True + 0 + + + + + True + False + 5 + 5 + 5 + + + False + True + 1 + + + + + + + + + True + False + Progress + + + + + 0 + 2 + + + + + + + + + True + False + Translation + + + + + 0 + 1 + 5 + + + + + + diff --git a/Gui test/h2inc.glade b/Gui test/h2inc.glade new file mode 100644 index 0000000..ad6a6e9 --- /dev/null +++ b/Gui test/h2inc.glade @@ -0,0 +1,367 @@ + + + + + + False + 5 + 5 + 5 + 5 + + + True + False + 5 + 5 + 5 + 5 + True + True + + + True + False + 0 + etched-out + + + True + False + 12 + + + True + False + + + True + False + end + 5 + 5 + 5 + True + Source: + right + + + 0 + 0 + 2 + + + + + True + True + 5 + 5 + 5 + 5 + Select source directory! + + + 2 + 0 + 2 + + + + + Source directory... + True + True + True + start + 5 + 5 + 5 + 5 + + + 4 + 0 + 2 + + + + + True + False + False + end + 5 + 5 + 5 + True + Destination: + right + start + 2.2351741811588166e-10 + + + 0 + 1 + 2 + + + + + True + False + True + 5 + 5 + 5 + 5 + Select destination directory! + + + 2 + 1 + 2 + + + + + Destination directory... + True + False + True + True + start + 5 + 5 + 5 + 5 + + + 4 + 1 + 2 + + + + + Create "include" folder if it does not exist. + True + False + True + False + 5 + 5 + 0 + True + + + 0 + 2 + 6 + + + + + + + + + True + False + Select folders + + + + + 0 + 0 + 5 + + + + + True + False + False + 5 + 5 + 0 + etched-out + + + True + False + 12 + + + True + False + + + True + False + 5 + 5 + True + 0 + etched-out + + + True + False + 12 + + + True + False + vertical + + + True + False + start + 5 + 5 + True + Number of folders: 0 + + + False + True + 0 + + + + + True + False + start + 5 + 5 + Number of files: 0 + + + False + True + 1 + + + + + + + + + True + False + Source information + + + + + 0 + 0 + + + + + Translate! + True + True + True + start + 5 + 5 + + + 0 + 1 + + + + + True + False + 5 + 5 + 5 + 0 + etched-out + + + True + False + 12 + + + True + False + vertical + + + True + False + start + 5 + 5 + Total progress: + + + False + True + 0 + + + + + True + False + 5 + 5 + 5 + + + False + True + 1 + + + + + + + + + True + False + Progress + + + + + 0 + 2 + + + + + + + + + True + False + Translation + + + + + 0 + 1 + 5 + + + + + + diff --git a/Gui test/h2inc.glade~ b/Gui test/h2inc.glade~ new file mode 100644 index 0000000..ad6a6e9 --- /dev/null +++ b/Gui test/h2inc.glade~ @@ -0,0 +1,367 @@ + + + + + + False + 5 + 5 + 5 + 5 + + + True + False + 5 + 5 + 5 + 5 + True + True + + + True + False + 0 + etched-out + + + True + False + 12 + + + True + False + + + True + False + end + 5 + 5 + 5 + True + Source: + right + + + 0 + 0 + 2 + + + + + True + True + 5 + 5 + 5 + 5 + Select source directory! + + + 2 + 0 + 2 + + + + + Source directory... + True + True + True + start + 5 + 5 + 5 + 5 + + + 4 + 0 + 2 + + + + + True + False + False + end + 5 + 5 + 5 + True + Destination: + right + start + 2.2351741811588166e-10 + + + 0 + 1 + 2 + + + + + True + False + True + 5 + 5 + 5 + 5 + Select destination directory! + + + 2 + 1 + 2 + + + + + Destination directory... + True + False + True + True + start + 5 + 5 + 5 + 5 + + + 4 + 1 + 2 + + + + + Create "include" folder if it does not exist. + True + False + True + False + 5 + 5 + 0 + True + + + 0 + 2 + 6 + + + + + + + + + True + False + Select folders + + + + + 0 + 0 + 5 + + + + + True + False + False + 5 + 5 + 0 + etched-out + + + True + False + 12 + + + True + False + + + True + False + 5 + 5 + True + 0 + etched-out + + + True + False + 12 + + + True + False + vertical + + + True + False + start + 5 + 5 + True + Number of folders: 0 + + + False + True + 0 + + + + + True + False + start + 5 + 5 + Number of files: 0 + + + False + True + 1 + + + + + + + + + True + False + Source information + + + + + 0 + 0 + + + + + Translate! + True + True + True + start + 5 + 5 + + + 0 + 1 + + + + + True + False + 5 + 5 + 5 + 0 + etched-out + + + True + False + 12 + + + True + False + vertical + + + True + False + start + 5 + 5 + Total progress: + + + False + True + 0 + + + + + True + False + 5 + 5 + 5 + + + False + True + 1 + + + + + + + + + True + False + Progress + + + + + 0 + 2 + + + + + + + + + True + False + Translation + + + + + 0 + 1 + 5 + + + + + + diff --git a/h2inc.glade b/h2inc.glade index 4702fd7..ad6a6e9 100644 --- a/h2inc.glade +++ b/h2inc.glade @@ -2,38 +2,6 @@ - - False - normal - select-folder - - - False - vertical - 2 - - - False - end - - - - - - - - - False - False - 0 - - - - - - - - False 5 @@ -44,6 +12,12 @@ True False + 5 + 5 + 5 + 5 + True + True True diff --git a/h2inc.glade~ b/h2inc.glade~ new file mode 100644 index 0000000..d0e629e --- /dev/null +++ b/h2inc.glade~ @@ -0,0 +1,365 @@ + + + + + + False + 5 + 5 + 5 + 5 + + + True + False + 5 + 5 + 5 + 5 + + + True + False + 0 + etched-out + + + True + False + 12 + + + True + False + + + True + False + end + 5 + 5 + 5 + True + Source: + right + + + 0 + 0 + 2 + + + + + True + True + 5 + 5 + 5 + 5 + Select source directory! + + + 2 + 0 + 2 + + + + + Source directory... + True + True + True + start + 5 + 5 + 5 + 5 + + + 4 + 0 + 2 + + + + + True + False + False + end + 5 + 5 + 5 + True + Destination: + right + start + 2.2351741811588166e-10 + + + 0 + 1 + 2 + + + + + True + False + True + 5 + 5 + 5 + 5 + Select destination directory! + + + 2 + 1 + 2 + + + + + Destination directory... + True + False + True + True + start + 5 + 5 + 5 + 5 + + + 4 + 1 + 2 + + + + + Create "include" folder if it does not exist. + True + False + True + False + 5 + 5 + 0 + True + + + 0 + 2 + 6 + + + + + + + + + True + False + Select folders + + + + + 0 + 0 + 5 + + + + + True + False + False + 5 + 5 + 0 + etched-out + + + True + False + 12 + + + True + False + + + True + False + 5 + 5 + True + 0 + etched-out + + + True + False + 12 + + + True + False + vertical + + + True + False + start + 5 + 5 + True + Number of folders: 0 + + + False + True + 0 + + + + + True + False + start + 5 + 5 + Number of files: 0 + + + False + True + 1 + + + + + + + + + True + False + Source information + + + + + 0 + 0 + + + + + Translate! + True + True + True + start + 5 + 5 + + + 0 + 1 + + + + + True + False + 5 + 5 + 5 + 0 + etched-out + + + True + False + 12 + + + True + False + vertical + + + True + False + start + 5 + 5 + Total progress: + + + False + True + 0 + + + + + True + False + 5 + 5 + 5 + + + False + True + 1 + + + + + + + + + True + False + Progress + + + + + 0 + 2 + + + + + + + + + True + False + Translation + + + + + 0 + 1 + 5 + + + + + + From ce436c8484ad5fcaad633f83ed0fd359aa4f7068 Mon Sep 17 00:00:00 2001 From: Jan Lerking Date: Thu, 22 Feb 2018 16:06:00 +0100 Subject: [PATCH 22/37] glade_v2 added --- Gui test/gtk3_glade_test.py | 2 +- Gui test/h2inc (copy).xml | 393 ---------------------- Gui test/{h2inc.glade~ => h2inc_v2.glade} | 38 +-- 3 files changed, 11 insertions(+), 422 deletions(-) delete mode 100644 Gui test/h2inc (copy).xml rename Gui test/{h2inc.glade~ => h2inc_v2.glade} (92%) diff --git a/Gui test/gtk3_glade_test.py b/Gui test/gtk3_glade_test.py index fad456f..06f6b30 100644 --- a/Gui test/gtk3_glade_test.py +++ b/Gui test/gtk3_glade_test.py @@ -10,7 +10,7 @@ class Handler: print("Hello World!") builder = Gtk.Builder() -builder.add_from_file("h2inc.glade") +builder.add_from_file("h2inc_v2.glade") builder.connect_signals(Handler()) window = builder.get_object("window1") diff --git a/Gui test/h2inc (copy).xml b/Gui test/h2inc (copy).xml deleted file mode 100644 index 4702fd7..0000000 --- a/Gui test/h2inc (copy).xml +++ /dev/null @@ -1,393 +0,0 @@ - - - - - - False - normal - select-folder - - - False - vertical - 2 - - - False - end - - - - - - - - - False - False - 0 - - - - - - - - - - False - 5 - 5 - 5 - 5 - - - True - False - - - True - False - 0 - etched-out - - - True - False - 12 - - - True - False - - - True - False - end - 5 - 5 - 5 - True - Source: - right - - - 0 - 0 - 2 - - - - - True - True - 5 - 5 - 5 - 5 - Select source directory! - - - 2 - 0 - 2 - - - - - Source directory... - True - True - True - start - 5 - 5 - 5 - 5 - - - 4 - 0 - 2 - - - - - True - False - False - end - 5 - 5 - 5 - True - Destination: - right - start - 2.2351741811588166e-10 - - - 0 - 1 - 2 - - - - - True - False - True - 5 - 5 - 5 - 5 - Select destination directory! - - - 2 - 1 - 2 - - - - - Destination directory... - True - False - True - True - start - 5 - 5 - 5 - 5 - - - 4 - 1 - 2 - - - - - Create "include" folder if it does not exist. - True - False - True - False - 5 - 5 - 0 - True - - - 0 - 2 - 6 - - - - - - - - - True - False - Select folders - - - - - 0 - 0 - 5 - - - - - True - False - False - 5 - 5 - 0 - etched-out - - - True - False - 12 - - - True - False - - - True - False - 5 - 5 - True - 0 - etched-out - - - True - False - 12 - - - True - False - vertical - - - True - False - start - 5 - 5 - True - Number of folders: 0 - - - False - True - 0 - - - - - True - False - start - 5 - 5 - Number of files: 0 - - - False - True - 1 - - - - - - - - - True - False - Source information - - - - - 0 - 0 - - - - - Translate! - True - True - True - start - 5 - 5 - - - 0 - 1 - - - - - True - False - 5 - 5 - 5 - 0 - etched-out - - - True - False - 12 - - - True - False - vertical - - - True - False - start - 5 - 5 - Total progress: - - - False - True - 0 - - - - - True - False - 5 - 5 - 5 - - - False - True - 1 - - - - - - - - - True - False - Progress - - - - - 0 - 2 - - - - - - - - - True - False - Translation - - - - - 0 - 1 - 5 - - - - - - diff --git a/Gui test/h2inc.glade~ b/Gui test/h2inc_v2.glade similarity index 92% rename from Gui test/h2inc.glade~ rename to Gui test/h2inc_v2.glade index ad6a6e9..74daa33 100644 --- a/Gui test/h2inc.glade~ +++ b/Gui test/h2inc_v2.glade @@ -9,15 +9,10 @@ 5 5 - + True False - 5 - 5 - 5 - 5 - True - True + vertical True @@ -38,7 +33,6 @@ True False end - 5 5 5 True @@ -55,8 +49,6 @@ True True - 5 - 5 5 5 Select source directory! @@ -74,8 +66,6 @@ True True start - 5 - 5 5 5 @@ -91,7 +81,6 @@ False False end - 5 5 5 True @@ -111,8 +100,6 @@ True False True - 5 - 5 5 5 Select destination directory! @@ -131,8 +118,6 @@ True True start - 5 - 5 5 5 @@ -173,9 +158,9 @@ - 0 - 0 - 5 + False + True + 0 @@ -200,7 +185,6 @@ True False - 5 5 True 0 @@ -211,7 +195,7 @@ False 12 - + True False vertical @@ -282,7 +266,6 @@ True False - 5 5 5 0 @@ -293,7 +276,7 @@ False 12 - + True False vertical @@ -316,7 +299,6 @@ True False - 5 5 5 @@ -356,9 +338,9 @@ - 0 - 1 - 5 + False + True + 1 From e1f39eb322acd749b54aa07b3ce4aef2c7820ac0 Mon Sep 17 00:00:00 2001 From: Jan Lerking Date: Thu, 22 Feb 2018 16:11:20 +0100 Subject: [PATCH 23/37] updated h2inc glade files --- Gui test/h2inc.glade | 39 +- Gui test/h2inc_orig.glade | 367 +++++++++++++++++++ Gui test/{h2inc_v2.glade => h2inc_v2.glade~} | 0 3 files changed, 378 insertions(+), 28 deletions(-) create mode 100644 Gui test/h2inc_orig.glade rename Gui test/{h2inc_v2.glade => h2inc_v2.glade~} (100%) diff --git a/Gui test/h2inc.glade b/Gui test/h2inc.glade index ad6a6e9..d8e6c2b 100644 --- a/Gui test/h2inc.glade +++ b/Gui test/h2inc.glade @@ -9,15 +9,10 @@ 5 5 - + True False - 5 - 5 - 5 - 5 - True - True + vertical True @@ -38,7 +33,6 @@ True False end - 5 5 5 True @@ -55,8 +49,6 @@ True True - 5 - 5 5 5 Select source directory! @@ -74,8 +66,6 @@ True True start - 5 - 5 5 5 @@ -91,7 +81,6 @@ False False end - 5 5 5 True @@ -111,8 +100,6 @@ True False True - 5 - 5 5 5 Select destination directory! @@ -131,8 +118,6 @@ True True start - 5 - 5 5 5 @@ -152,6 +137,7 @@ 5 5 0 + True True @@ -173,9 +159,9 @@ - 0 - 0 - 5 + False + True + 0 @@ -200,7 +186,6 @@ True False - 5 5 True 0 @@ -211,7 +196,7 @@ False 12 - + True False vertical @@ -282,7 +267,6 @@ True False - 5 5 5 0 @@ -293,7 +277,7 @@ False 12 - + True False vertical @@ -316,7 +300,6 @@ True False - 5 5 5 @@ -356,9 +339,9 @@ - 0 - 1 - 5 + False + True + 1 diff --git a/Gui test/h2inc_orig.glade b/Gui test/h2inc_orig.glade new file mode 100644 index 0000000..ad6a6e9 --- /dev/null +++ b/Gui test/h2inc_orig.glade @@ -0,0 +1,367 @@ + + + + + + False + 5 + 5 + 5 + 5 + + + True + False + 5 + 5 + 5 + 5 + True + True + + + True + False + 0 + etched-out + + + True + False + 12 + + + True + False + + + True + False + end + 5 + 5 + 5 + True + Source: + right + + + 0 + 0 + 2 + + + + + True + True + 5 + 5 + 5 + 5 + Select source directory! + + + 2 + 0 + 2 + + + + + Source directory... + True + True + True + start + 5 + 5 + 5 + 5 + + + 4 + 0 + 2 + + + + + True + False + False + end + 5 + 5 + 5 + True + Destination: + right + start + 2.2351741811588166e-10 + + + 0 + 1 + 2 + + + + + True + False + True + 5 + 5 + 5 + 5 + Select destination directory! + + + 2 + 1 + 2 + + + + + Destination directory... + True + False + True + True + start + 5 + 5 + 5 + 5 + + + 4 + 1 + 2 + + + + + Create "include" folder if it does not exist. + True + False + True + False + 5 + 5 + 0 + True + + + 0 + 2 + 6 + + + + + + + + + True + False + Select folders + + + + + 0 + 0 + 5 + + + + + True + False + False + 5 + 5 + 0 + etched-out + + + True + False + 12 + + + True + False + + + True + False + 5 + 5 + True + 0 + etched-out + + + True + False + 12 + + + True + False + vertical + + + True + False + start + 5 + 5 + True + Number of folders: 0 + + + False + True + 0 + + + + + True + False + start + 5 + 5 + Number of files: 0 + + + False + True + 1 + + + + + + + + + True + False + Source information + + + + + 0 + 0 + + + + + Translate! + True + True + True + start + 5 + 5 + + + 0 + 1 + + + + + True + False + 5 + 5 + 5 + 0 + etched-out + + + True + False + 12 + + + True + False + vertical + + + True + False + start + 5 + 5 + Total progress: + + + False + True + 0 + + + + + True + False + 5 + 5 + 5 + + + False + True + 1 + + + + + + + + + True + False + Progress + + + + + 0 + 2 + + + + + + + + + True + False + Translation + + + + + 0 + 1 + 5 + + + + + + diff --git a/Gui test/h2inc_v2.glade b/Gui test/h2inc_v2.glade~ similarity index 100% rename from Gui test/h2inc_v2.glade rename to Gui test/h2inc_v2.glade~ From 524038a0c3bc35e8abec53bf250558f53dad817e Mon Sep 17 00:00:00 2001 From: Jan Lerking Date: Fri, 23 Feb 2018 12:02:16 +0100 Subject: [PATCH 24/37] added some more glade testing --- Gui test/16_filechooser.glade | 145 ++++++++ Gui test/16_filechooser.py | 95 +++++ Gui test/gtk3_glade_test.py | 16 +- Gui test/test.glade | 433 ++++++++++++++++++++++ Gui test/{h2inc_v2.glade~ => test.glade~} | 299 ++++++++++----- 5 files changed, 884 insertions(+), 104 deletions(-) create mode 100644 Gui test/16_filechooser.glade create mode 100644 Gui test/16_filechooser.py create mode 100644 Gui test/test.glade rename Gui test/{h2inc_v2.glade~ => test.glade~} (63%) diff --git a/Gui test/16_filechooser.glade b/Gui test/16_filechooser.glade new file mode 100644 index 0000000..d867d52 --- /dev/null +++ b/Gui test/16_filechooser.glade @@ -0,0 +1,145 @@ + + + + + + + image/* + + + + + image/jpeg + + + + + image/png + + + + 200 + True + False + 5 + + + 300 + 200 + False + + + + True + False + vertical + True + + + Choose an image file... + True + True + True + + + + False + True + 0 + + + + + Choose folder... + True + True + True + + + + False + True + 1 + + + + + + + + + + 800 + 500 + False + dialog + window + window + preview + False + + + + + + + False + vertical + + + False + + + gtk-cancel + True + True + True + True + + + True + True + 2 + + + + + gtk-apply + True + True + True + True + + + True + True + 3 + + + + + False + False + 0 + + + + + + button2 + button1 + + + + True + False + Choose image... + True + + + + button2 + button1 + + + diff --git a/Gui test/16_filechooser.py b/Gui test/16_filechooser.py new file mode 100644 index 0000000..f8be92c --- /dev/null +++ b/Gui test/16_filechooser.py @@ -0,0 +1,95 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +import os +import sys +import gi +gi.require_version('Gtk', '3.0') +from gi.repository import Gtk, Gio, GdkPixbuf + +class Handler: + + def on_window_destroy(self,window): + window.close() + + def on_dialog_close(self,widget,*event): + widget.hide_on_delete() + return True + + def on_filechooser_dialog_response(self,widget,response): + if response == -6: + print("Cancel") + elif response == -5: + print("File selection: %s" % widget.get_filename()) + self.on_dialog_close(widget) + + def on_filechooser_dialog_file_activated(self,widget): + self.on_filechooser_dialog_response(widget,-5) + + def on_filechooser_dialog_update_preview(self,widget): + if widget.get_filename() != None and os.path.isfile(widget.get_filename()): + pixbuf = GdkPixbuf.Pixbuf.new_from_file_at_scale(widget.get_filename(),200,200,True) + app.obj("preview").set_from_pixbuf(pixbuf) + + def on_file_button_clicked(self,widget): + app.obj("filechooser_dialog").show_all() + + def on_dir_button_clicked(self,widget): + + dialog = Gtk.FileChooserDialog("Choose a folder", + app.obj("window"), + Gtk.FileChooserAction.SELECT_FOLDER, + (Gtk.STOCK_CANCEL, Gtk.ResponseType.CANCEL, + Gtk.STOCK_APPLY, Gtk.ResponseType.OK)) + dialog.set_default_size(600, 300) + + response = dialog.run() + if response == Gtk.ResponseType.OK: + print("Folder selection: %s" % dialog.get_filename()) + elif response == Gtk.ResponseType.CANCEL: + print("Cancel") + + dialog.destroy() + +class ExampleApp: + + def __init__(self): + + self.app = Gtk.Application.new("org.application.test", Gio.ApplicationFlags(0)) + self.app.connect("activate", self.on_app_activate) + self.app.connect("shutdown", self.on_app_shutdown) + + def on_app_activate(self, app): + builder = Gtk.Builder() + builder.add_from_file("16_filechooser.glade") + builder.connect_signals(Handler()) + + self.obj = builder.get_object + self.obj("window").set_application(app) + self.obj("window").set_wmclass("Filechooser example","Filechooser example") + self.obj("window").show_all() + + #add filters to filechooser dialog + self.obj("filefilter").set_name("Image files") + self.obj("filechooser_dialog").add_filter(self.obj("filefilter")) + self.obj("png_filter").set_name("PNG files") + self.obj("filechooser_dialog").add_filter(self.obj("png_filter")) + self.obj("jpg_filter").set_name("JPG files") + self.obj("filechooser_dialog").add_filter(self.obj("jpg_filter")) + + #add buttons to headerbar of Glade generated dialog + button = Gtk.Button.new_from_stock(Gtk.STOCK_CANCEL) + button.set_property("can-default",True) + self.obj("filechooser_dialog").add_action_widget(button, Gtk.ResponseType.CANCEL) + button = Gtk.Button.new_from_stock(Gtk.STOCK_APPLY) + button.set_property("can-default",True) + self.obj("filechooser_dialog").add_action_widget(button, Gtk.ResponseType.OK) + + def on_app_shutdown(self, app): + self.app.quit() + + def run(self, argv): + self.app.run(argv) + +app = ExampleApp() +app.run(sys.argv) diff --git a/Gui test/gtk3_glade_test.py b/Gui test/gtk3_glade_test.py index 06f6b30..f690bba 100644 --- a/Gui test/gtk3_glade_test.py +++ b/Gui test/gtk3_glade_test.py @@ -6,13 +6,23 @@ class Handler: def onDeleteWindow(self, *args): Gtk.main_quit(*args) - def onButtonPressed(self, button): - print("Hello World!") + def on_button1_clicked(self, button): + dialog.set_title('Select source folder') + dialog.set_default_size(800, 400) + + response = dialog.run() + if response == Gtk.ResponseType.OK: + print("Folder selected: " + dialog.get_filename()) + elif response == Gtk.ResponseType.CANCEL: + print("Cancel clicked") + + dialog.destroy() builder = Gtk.Builder() -builder.add_from_file("h2inc_v2.glade") +builder.add_from_file("test.glade") builder.connect_signals(Handler()) +dialog = builder.get_object("filechooserdialog1") window = builder.get_object("window1") window.connect('delete-event', Gtk.main_quit) window.show_all() diff --git a/Gui test/test.glade b/Gui test/test.glade new file mode 100644 index 0000000..8cb0a99 --- /dev/null +++ b/Gui test/test.glade @@ -0,0 +1,433 @@ + + + + + + False + + + True + False + vertical + + + True + False + 5 + 0 + etched-out + + + True + False + 12 + + + True + False + + + True + False + end + 5 + 5 + 5 + Source: + start + + + 0 + 0 + + + + + True + False + False + end + 5 + 5 + 5 + Destination: + 0.059999999999999998 + + + 0 + 1 + + + + + True + True + 5 + 5 + + + 1 + 0 + + + + + True + False + True + 5 + 5 + + + 1 + 1 + + + + + Select source directory... + True + True + True + start + 5 + 5 + 0.52999997138977051 + + + 2 + 0 + + + + + Select destination directory... + True + False + True + True + start + 5 + 5 + + + 2 + 1 + + + + + checkbutton + True + False + True + False + 0 + True + True + + + 0 + 2 + 3 + + + + + + + + + True + False + Select folders + + + + + True + True + 0 + + + + + True + False + False + 5 + 0 + etched-out + + + True + False + 12 + + + True + False + vertical + + + True + False + 5 + 0 + etched-out + + + True + False + 12 + + + True + False + + + True + False + end + 5 + Number of folders: + + + 0 + 0 + + + + + True + False + 0.9882352941176471 + start + 5 + 0 + end + + + 1 + 0 + + + + + True + False + end + 5 + 5 + Number of files: + + + 0 + 1 + + + + + True + False + start + 5 + 5 + 0 + + + 1 + 1 + + + + + + + + + True + False + Source information + + + + + False + True + 0 + + + + + Translate! + True + True + True + start + 5 + 5 + + + False + False + 1 + + + + + True + False + 5 + 0 + etched-out + + + True + False + 12 + + + True + False + + + True + False + end + 5 + 5 + Total progress: + + + 0 + 0 + + + + + True + False + start + 5 + 5 + True + 0 of 0 + + + 1 + 0 + + + + + True + False + 5 + + + 0 + 1 + 2 + + + + + + + + + True + False + Progress + + + + + False + True + 2 + + + + + + + + + True + False + Translation! + + + + + True + True + 1 + + + + + + + False + dialog + select-folder + + + False + vertical + 2 + + + False + end + + + gtk-ok + True + True + True + True + True + + + True + True + 0 + + + + + gtk-cancel + True + True + True + True + True + + + True + True + 1 + + + + + False + False + 0 + + + + + + + + + button4 + button5 + + + diff --git a/Gui test/h2inc_v2.glade~ b/Gui test/test.glade~ similarity index 63% rename from Gui test/h2inc_v2.glade~ rename to Gui test/test.glade~ index 74daa33..a48e0df 100644 --- a/Gui test/h2inc_v2.glade~ +++ b/Gui test/test.glade~ @@ -2,21 +2,80 @@ + + False + dialog + select-folder + + + False + vertical + 2 + + + False + end + + + gtk-ok + True + True + True + True + True + + + True + True + 0 + + + + + gtk-cancel + True + True + True + True + True + + + True + True + 1 + + + + + False + False + 0 + + + + + + + + + button4 + button5 + + False - 5 - 5 - 5 - 5 True False vertical - + True False + 5 + 5 + 5 0 etched-out @@ -25,24 +84,40 @@ False 12 - + True False - + True False end 5 5 - True + 5 Source: - right + start 0 0 - 2 + + + + + True + False + False + end + 5 + 5 + 5 + Destination: + 0.059999999999999998 + + + 0 + 1 @@ -51,48 +126,10 @@ True 5 5 - Select source directory! - 2 + 1 0 - 2 - - - - - Source directory... - True - True - True - start - 5 - 5 - - - 4 - 0 - 2 - - - - - True - False - False - end - 5 - 5 - True - Destination: - right - start - 2.2351741811588166e-10 - - - 0 - 1 - 2 @@ -102,47 +139,65 @@ True 5 5 - Select destination directory! + + + 1 + 1 + + + + + Select source directory... + True + True + True + start + 5 + 5 + 5 + 0.52999997138977051 + 2 - 1 - 2 + 0 - Destination directory... + Select destination directory... True False True True start + 5 + 10 5 5 + - 4 + 2 1 - 2 - Create "include" folder if it does not exist. + checkbutton True False True False - 5 - 5 0 + 0.47999998927116394 + True True 0 2 - 6 + 3 @@ -158,17 +213,18 @@ - False + True True 0 - + True False False - 5 + 5 + 5 5 0 etched-out @@ -178,15 +234,16 @@ False 12 - + True False + vertical - + True False + 10 5 - True 0 etched-out @@ -195,39 +252,63 @@ False 12 - + True False - vertical True False - start + end 5 - 5 - True - Number of folders: 0 + Number of folders: - False - True - 0 + 0 + 0 - + + True + False + 0.98999999999999999 + start + 5 + 0 + end + + + 1 + 0 + + + + + True + False + end + 5 + 5 + Number of files: + + + 0 + 1 + + + + True False start 5 5 - Number of files: 0 + 0 - False - True - 1 + 1 + 1 @@ -243,8 +324,9 @@ - 0 - 0 + False + True + 0 @@ -258,15 +340,16 @@ 5 - 0 - 1 + False + False + 1 - + True False - 5 + 10 5 0 etched-out @@ -276,36 +359,49 @@ False 12 - + True False - vertical + True + False + end + 5 + 5 + Total progress: + + + 0 + 0 + + + + True False start 5 5 - Total progress: + True + 0 of 0 - False - True - 0 + 1 + 0 True False - 5 + 10 5 - False - True - 1 + 0 + 1 + 2 @@ -313,7 +409,7 @@ - + True False Progress @@ -321,8 +417,9 @@ - 0 - 2 + False + True + 2 @@ -330,15 +427,15 @@ - + True False - Translation + Translation! - False + True True 1 From 6189a82a7daf236610e60d02aa673b78201e8041 Mon Sep 17 00:00:00 2001 From: Jan Lerking Date: Sat, 24 Feb 2018 22:04:15 +0100 Subject: [PATCH 25/37] update h2inc.glade --- .idea/h2inc.iml | 12 +++++ .idea/misc.xml | 4 ++ .idea/modules.xml | 8 ++++ Gui test/h2inc.glade | 77 +++++++++++++++++++++++++++---- Gui test/h2inc_gtk.py | 102 ++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 194 insertions(+), 9 deletions(-) create mode 100644 .idea/h2inc.iml create mode 100644 .idea/misc.xml create mode 100644 .idea/modules.xml create mode 100644 Gui test/h2inc_gtk.py diff --git a/.idea/h2inc.iml b/.idea/h2inc.iml new file mode 100644 index 0000000..1de84ce --- /dev/null +++ b/.idea/h2inc.iml @@ -0,0 +1,12 @@ + + + + + + + + + + \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml new file mode 100644 index 0000000..7ba73c2 --- /dev/null +++ b/.idea/misc.xml @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 0000000..0cc5137 --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/Gui test/h2inc.glade b/Gui test/h2inc.glade index d8e6c2b..b1f9dbe 100644 --- a/Gui test/h2inc.glade +++ b/Gui test/h2inc.glade @@ -2,12 +2,41 @@ - + + False + dialog + + + False + vertical + 2 + + + False + + + + + + + + + + + + False + False + 0 + + + + + + + + + False - 5 - 5 - 5 - 5 True @@ -17,6 +46,10 @@ True False + 5 + 5 + 5 + 5 0 etched-out @@ -33,6 +66,8 @@ True False end + 5 + 5 5 5 True @@ -60,14 +95,17 @@ - - Source directory... + + Select source directory... True True True start + 5 + 5 5 5 + 4 @@ -81,6 +119,8 @@ False False end + 5 + 5 5 5 True @@ -111,15 +151,18 @@ - - Destination directory... + + Select destination directory... True False True True start + 5 + 5 5 5 + 4 @@ -134,6 +177,8 @@ False True False + 5 + 5 5 5 0 @@ -154,6 +199,8 @@ True False + 5 + 5 Select folders @@ -169,6 +216,8 @@ True False False + 5 + 5 5 5 0 @@ -186,6 +235,8 @@ True False + 5 + 5 5 True 0 @@ -239,6 +290,8 @@ True False + 5 + 5 Source information @@ -267,6 +320,7 @@ True False + 5 5 5 0 @@ -300,6 +354,7 @@ True False + 5 5 5 @@ -317,6 +372,8 @@ True False + 5 + 5 Progress @@ -334,6 +391,8 @@ True False + 5 + 5 Translation diff --git a/Gui test/h2inc_gtk.py b/Gui test/h2inc_gtk.py new file mode 100644 index 0000000..290b100 --- /dev/null +++ b/Gui test/h2inc_gtk.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python3.5 + +# You are free to use and/or change this code for +# your own needs. + +# Original code (c)2018 Jan Lerking +# Program to convert C-header (*.h) files to nasm include files (*.inc), +# for direct usage in assembly programming using nasm/yasm. + +import os +import sys +import gi +gi.require_version('Gtk', '3.0') +from gi.repository import Gtk, Gio, GdkPixbuf + +class Handler: + + def on_window_destroy(self,window): + window.close() + + def on_dialog_close(self,widget,*event): + widget.hide_on_delete() + return True + + def on_filechooser_dialog_response(self,widget,response): + if response == -6: + print("Cancel") + elif response == -5: + print("File selection: %s" % widget.get_filename()) + self.on_dialog_close(widget) + + def on_filechooser_dialog_file_activated(self,widget): + self.on_filechooser_dialog_response(widget,-5) + + def on_filechooser_dialog_update_preview(self,widget): + if widget.get_filename() != None and os.path.isfile(widget.get_filename()): + pixbuf = GdkPixbuf.Pixbuf.new_from_file_at_scale(widget.get_filename(),200,200,True) + app.obj("preview").set_from_pixbuf(pixbuf) + + def on_file_button_clicked(self,widget): + app.obj("filechooser_dialog").show_all() + + def on_source_button_clicked(self,widget): + + dialog = Gtk.FileChooserDialog("Select source directory!", + app.obj("window"), + Gtk.FileChooserAction.SELECT_FOLDER, + (Gtk.STOCK_CANCEL, Gtk.ResponseType.CANCEL, + Gtk.STOCK_APPLY, Gtk.ResponseType.OK)) + dialog.set_default_size(600, 300) + + response = dialog.run() + if response == Gtk.ResponseType.OK: + print("Folder selection: %s" % dialog.get_filename()) + elif response == Gtk.ResponseType.CANCEL: + print("Cancel") + + dialog.destroy() + +class ExampleApp: + + def __init__(self): + + self.app = Gtk.Application.new("org.h2inc", Gio.ApplicationFlags(0)) + self.app.connect("activate", self.on_app_activate) + self.app.connect("shutdown", self.on_app_shutdown) + + def on_app_activate(self, app): + builder = Gtk.Builder() + #builder.add_from_file("16_filechooser.glade") + builder.add_from_file("h2inc.glade") + builder.connect_signals(Handler()) + + self.obj = builder.get_object + self.obj("window").set_application(app) + self.obj("window").set_wmclass("Filechooser example","Filechooser example") + self.obj("window").show_all() + + #add filters to filechooser dialog + self.obj("filefilter").set_name("Image files") + self.obj("filechooser_dialog").add_filter(self.obj("filefilter")) + self.obj("png_filter").set_name("PNG files") + self.obj("filechooser_dialog").add_filter(self.obj("png_filter")) + self.obj("jpg_filter").set_name("JPG files") + self.obj("filechooser_dialog").add_filter(self.obj("jpg_filter")) + + #add buttons to headerbar of Glade generated dialog + button = Gtk.Button.new_from_stock(Gtk.STOCK_CANCEL) + button.set_property("can-default",True) + self.obj("filechooser_dialog").add_action_widget(button, Gtk.ResponseType.CANCEL) + button = Gtk.Button.new_from_stock(Gtk.STOCK_APPLY) + button.set_property("can-default",True) + self.obj("filechooser_dialog").add_action_widget(button, Gtk.ResponseType.OK) + + def on_app_shutdown(self, app): + self.app.quit() + + def run(self, argv): + self.app.run(argv) + +app = ExampleApp() +app.run(sys.argv) From 1bf8c523fe228d406a77bfa661e8c49368aafa0e Mon Sep 17 00:00:00 2001 From: Jan Lerking Date: Sat, 24 Feb 2018 22:11:04 +0100 Subject: [PATCH 26/37] updated h2inc_gtk.py --- Gui test/h2inc_gtk.py | 106 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 105 insertions(+), 1 deletion(-) diff --git a/Gui test/h2inc_gtk.py b/Gui test/h2inc_gtk.py index 290b100..ac145dc 100644 --- a/Gui test/h2inc_gtk.py +++ b/Gui test/h2inc_gtk.py @@ -9,10 +9,114 @@ import os import sys +import multiprocessing as mp +import io import gi gi.require_version('Gtk', '3.0') from gi.repository import Gtk, Gio, GdkPixbuf +tupline = [] +preproc = () +filelist = [] +folderlist = [] +cnt = 0 +sourcedir = '' +destdir = '' +num_cores = mp.cpu_count() + + +def sourcedir_filecnt(sourcedir): + ### Return the number of files, ending with '.h', in sourcedir - including subdirectories ### + cnt = 0 + global filelist + for folderName, subfolders, files in os.walk(sourcedir): + for file in files: + if file.lower().endswith('.h'): + cnt += 1 + filelist += [folderName + '/' + file] + print(folderName + '/' + file) + # print(filelist) + return cnt + + +def sourcedir_foldercnt(sourcedir): + ### Return the number of folders, if it contains '*.h' files, in sourcedir - including subdirectories ### + global cnt + global folderlist + for folderName, subfolders, files in os.walk(sourcedir): + if subfolders: + for subfolder in subfolders: + sourcedir_foldercnt(subfolder) + tempf = [file for file in files if file.lower().endswith('.h')] + if tempf: + cnt = cnt + 1 + # print(folderName) + folderlist += [folderName] + # print(folderlist) + # print(len(folderlist)) + return cnt + + +def process_files(gui, source, dest): + global sourcedir + global destdir + sourcedir = source + destdir = dest + pool = mp.Pool(processes=num_cores) + pool.map(process_file, filelist) + + +def process_file(data): + outfile = '' + inputfile = data + encodings = ['utf-8', 'latin-1', 'windows-1250', 'windows-1252', 'ascii', + 'big5', 'big5hkscs', 'cp037', 'cp273', 'cp424', 'cp437', 'cp500', + 'cp720', 'cp737', 'cp775', 'cp850', 'cp852', 'cp855', 'cp856', + 'cp857', 'cp858', 'cp860', 'cp861', 'cp862', 'cp863', 'cp864', 'cp865', + 'cp866', 'cp869', 'cp874', 'cp875', 'cp932', 'cp949', 'cp950', 'cp1006', + 'cp1026', 'cp1125', 'cp1140', 'cp1250', 'cp1251', 'cp1252', 'cp1253', 'cp1254', + 'cp1255', 'cp1256', 'cp1257', 'cp1258', 'cp65001', 'euc-jp', 'euc-jis-2004', + 'euc-jisx0213', 'euc-kr', 'gb2312', 'gbk', 'gb18030', 'hz', 'iso2022-jp', + 'iso2022-jp-1', 'iso2022-jp-2', 'iso2022-jp-2004', 'iso2022-jp-3', + 'iso2022-jp-ext', 'iso2022-kr', 'iso8859-2', 'iso8859-3', 'iso8859-4', + 'iso8859-5', 'iso8859-6', 'iso8859-7', 'iso8859-8', 'iso8859-9', 'iso8859-10', + 'iso8859-11', 'iso8859-13', 'iso8859-14', 'iso8859-15', 'iso8859-16', 'johab', + 'koi8-r', 'koi8-t', 'koi8-u', 'kz1048', 'mac-cyrillic', 'mac-greek', + 'mac-iceland', 'mac-latin2', 'mac-roman', 'mac-turkish', 'ptcp154', + 'shift-jis', 'shift-jis-2004', 'shift-jisx0213', 'utf-32', 'utf-32-be', + 'utf-32-le', 'utf-16', 'utf-16-be', 'utf-16-le', 'utf-7', 'utf-8-sig'] + for e in encodings: + try: + fh = io.open(data, 'r', encoding=e) + fh.readlines() + fh.seek(0) + except UnicodeDecodeError: + print('got unicode error with %s , trying different encoding' % e) + else: + # print('opening the file with encoding: %s ' % e) + break + # print(os.path.basename(data)) + for lines in fh: + outfile = outfile + lines + fh.close() + outputfile = os.path.splitext(inputfile)[0] + '.inc' + outputfile = str(outputfile).replace(sourcedir, destdir) + # print(outputfile) + if not os.path.exists(os.path.dirname(outputfile)): + try: + os.makedirs(os.path.dirname(outputfile)) + except OSError as exc: # Guard against race condition + if exc.errno != errno.EEXIST: + raise + newfile = open(outputfile, "w") + newfile.write(outfile) + newfile.close() + + +def async_process(num): + pool = mp.Pool(processes=num) + pool.map(process_file, filelist) + class Handler: def on_window_destroy(self,window): @@ -51,7 +155,7 @@ class Handler: response = dialog.run() if response == Gtk.ResponseType.OK: - print("Folder selection: %s" % dialog.get_filename()) + sourcedir_filecnt(dialog.get_filename()) elif response == Gtk.ResponseType.CANCEL: print("Cancel") From 4b1ad6d614f19df4d8a4cb541997f9d1acc80e12 Mon Sep 17 00:00:00 2001 From: Jan Lerking Date: Sat, 24 Feb 2018 23:11:39 +0100 Subject: [PATCH 27/37] updated h2inc_gtk.py --- .idea/vcs.xml | 6 ++++++ Gui test/h2inc.glade | 10 +++++----- Gui test/h2inc_gtk.py | 21 ++++++++++++--------- 3 files changed, 23 insertions(+), 14 deletions(-) create mode 100644 .idea/vcs.xml diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 0000000..94a25f7 --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/Gui test/h2inc.glade b/Gui test/h2inc.glade index b1f9dbe..920ffb5 100644 --- a/Gui test/h2inc.glade +++ b/Gui test/h2inc.glade @@ -62,7 +62,7 @@ True False - + True False end @@ -81,7 +81,7 @@ - + True True 5 @@ -114,7 +114,7 @@ - + True False False @@ -136,7 +136,7 @@ - + True False True @@ -171,7 +171,7 @@ - + Create "include" folder if it does not exist. True False diff --git a/Gui test/h2inc_gtk.py b/Gui test/h2inc_gtk.py index ac145dc..c2eb4d2 100644 --- a/Gui test/h2inc_gtk.py +++ b/Gui test/h2inc_gtk.py @@ -155,7 +155,18 @@ class Handler: response = dialog.run() if response == Gtk.ResponseType.OK: - sourcedir_filecnt(dialog.get_filename()) + cnt = sourcedir_filecnt(dialog.get_filename()) + if cnt >0: + print(cnt) + builder = Gtk.Builder() + builder.add_from_file("h2inc.glade") + obj = builder.get_object + obj("destination_label").set_sensitive(True) + obj("destination_entry").set_sensitive(True) + obj("destination_button").set_sensitive(True) + obj("include_checkbutton").set_sensitive(True) + while Gtk.events_pending(): + Gtk.main_iteration() elif response == Gtk.ResponseType.CANCEL: print("Cancel") @@ -180,14 +191,6 @@ class ExampleApp: self.obj("window").set_wmclass("Filechooser example","Filechooser example") self.obj("window").show_all() - #add filters to filechooser dialog - self.obj("filefilter").set_name("Image files") - self.obj("filechooser_dialog").add_filter(self.obj("filefilter")) - self.obj("png_filter").set_name("PNG files") - self.obj("filechooser_dialog").add_filter(self.obj("png_filter")) - self.obj("jpg_filter").set_name("JPG files") - self.obj("filechooser_dialog").add_filter(self.obj("jpg_filter")) - #add buttons to headerbar of Glade generated dialog button = Gtk.Button.new_from_stock(Gtk.STOCK_CANCEL) button.set_property("can-default",True) From b434526b312a6949d6116faca56555e8f18c006f Mon Sep 17 00:00:00 2001 From: Jan Lerking Date: Sun, 25 Feb 2018 10:51:04 +0100 Subject: [PATCH 28/37] updated h2inc_gtk.py and h2inc.glade --- Gui test/h2inc.glade | 6 +++--- Gui test/h2inc_gtk.py | 41 ++++++++++++++++++++++++++++------------- 2 files changed, 31 insertions(+), 16 deletions(-) diff --git a/Gui test/h2inc.glade b/Gui test/h2inc.glade index 920ffb5..8db104c 100644 --- a/Gui test/h2inc.glade +++ b/Gui test/h2inc.glade @@ -252,7 +252,7 @@ False vertical - + True False start @@ -268,7 +268,7 @@ - + True False start @@ -369,7 +369,7 @@ - + True False 5 diff --git a/Gui test/h2inc_gtk.py b/Gui test/h2inc_gtk.py index c2eb4d2..262a0cb 100644 --- a/Gui test/h2inc_gtk.py +++ b/Gui test/h2inc_gtk.py @@ -24,7 +24,6 @@ sourcedir = '' destdir = '' num_cores = mp.cpu_count() - def sourcedir_filecnt(sourcedir): ### Return the number of files, ending with '.h', in sourcedir - including subdirectories ### cnt = 0 @@ -158,15 +157,29 @@ class Handler: cnt = sourcedir_filecnt(dialog.get_filename()) if cnt >0: print(cnt) - builder = Gtk.Builder() - builder.add_from_file("h2inc.glade") - obj = builder.get_object - obj("destination_label").set_sensitive(True) - obj("destination_entry").set_sensitive(True) - obj("destination_button").set_sensitive(True) - obj("include_checkbutton").set_sensitive(True) - while Gtk.events_pending(): - Gtk.main_iteration() + app.obj("source_entry").set_text(dialog.get_filename()) + app.obj("destination_label").set_sensitive(True) + app.obj("destination_entry").set_sensitive(True) + app.obj("destination_button").set_sensitive(True) + elif response == Gtk.ResponseType.CANCEL: + print("Cancel") + + dialog.destroy() + + def on_destination_button_clicked(self,widget): + + dialog = Gtk.FileChooserDialog("Select destination directory!", + app.obj("window"), + Gtk.FileChooserAction.SELECT_FOLDER, + (Gtk.STOCK_CANCEL, Gtk.ResponseType.CANCEL, + Gtk.STOCK_APPLY, Gtk.ResponseType.OK)) + dialog.set_default_size(600, 300) + + response = dialog.run() + if response == Gtk.ResponseType.OK: + destdir = dialog.get_filename() + app.obj("destination_entry").set_text(dialog.get_filename()) + app.obj("include_checkbutton").set_sensitive(True) elif response == Gtk.ResponseType.CANCEL: print("Cancel") @@ -174,24 +187,26 @@ class Handler: class ExampleApp: + global app + global destlabel + def __init__(self): self.app = Gtk.Application.new("org.h2inc", Gio.ApplicationFlags(0)) self.app.connect("activate", self.on_app_activate) self.app.connect("shutdown", self.on_app_shutdown) + app = self.app def on_app_activate(self, app): builder = Gtk.Builder() - #builder.add_from_file("16_filechooser.glade") builder.add_from_file("h2inc.glade") builder.connect_signals(Handler()) self.obj = builder.get_object self.obj("window").set_application(app) - self.obj("window").set_wmclass("Filechooser example","Filechooser example") + self.obj("window").set_wmclass("h2inc_gtk","h2inc_gtk") self.obj("window").show_all() - #add buttons to headerbar of Glade generated dialog button = Gtk.Button.new_from_stock(Gtk.STOCK_CANCEL) button.set_property("can-default",True) self.obj("filechooser_dialog").add_action_widget(button, Gtk.ResponseType.CANCEL) From 1a978407c056853f64f3b5c90b51cebd9041f3c7 Mon Sep 17 00:00:00 2001 From: Jan Lerking Date: Sun, 25 Feb 2018 12:30:23 +0100 Subject: [PATCH 29/37] updated h2inc_gtk.py and h2inc.glade --- Gui test/h2inc.glade | 106 +++++++++++++++++++++++++++++++----------- Gui test/h2inc_gtk.py | 46 ++++++++++++------ 2 files changed, 111 insertions(+), 41 deletions(-) diff --git a/Gui test/h2inc.glade b/Gui test/h2inc.glade index 8db104c..128c53f 100644 --- a/Gui test/h2inc.glade +++ b/Gui test/h2inc.glade @@ -212,7 +212,7 @@ - + True False False @@ -252,14 +252,33 @@ False vertical - + True False - start - 5 - 5 - True - Number of folders: 0 + + + True + False + 5 + 5 + Number of folders: + + + 0 + 0 + + + + + True + False + 0 + + + 1 + 0 + + False @@ -268,13 +287,33 @@ - + True False - start - 5 - 5 - Number of files: 0 + + + True + False + 5 + 5 + Number of files: + + + 0 + 0 + + + + + True + False + 0 + + + 1 + 0 + + False @@ -302,7 +341,7 @@ - + Translate! True True @@ -310,6 +349,7 @@ start 5 5 + 0 @@ -331,37 +371,49 @@ False 12 - + True False - vertical + True - + True False start - 5 - 5 - Total progress: + False + Total progress: - False - True - 0 + 0 + 0 - + + True + False + start + True + 0 of 0 + + + 1 + 0 + + + + True False 5 5 5 + True - False - True - 1 + 0 + 1 + 2 @@ -369,7 +421,7 @@ - + True False 5 diff --git a/Gui test/h2inc_gtk.py b/Gui test/h2inc_gtk.py index 262a0cb..48bcd28 100644 --- a/Gui test/h2inc_gtk.py +++ b/Gui test/h2inc_gtk.py @@ -20,14 +20,17 @@ preproc = () filelist = [] folderlist = [] cnt = 0 -sourcedir = '' +srcdir = '' destdir = '' num_cores = mp.cpu_count() +fileindex = 0 def sourcedir_filecnt(sourcedir): ### Return the number of files, ending with '.h', in sourcedir - including subdirectories ### cnt = 0 global filelist + global srcdir + srcdir = sourcedir for folderName, subfolders, files in os.walk(sourcedir): for file in files: if file.lower().endswith('.h'): @@ -55,16 +58,12 @@ def sourcedir_foldercnt(sourcedir): # print(len(folderlist)) return cnt - -def process_files(gui, source, dest): - global sourcedir - global destdir - sourcedir = source - destdir = dest +def process_files(): + print(srcdir) + print(destdir) pool = mp.Pool(processes=num_cores) pool.map(process_file, filelist) - def process_file(data): outfile = '' inputfile = data @@ -99,8 +98,8 @@ def process_file(data): outfile = outfile + lines fh.close() outputfile = os.path.splitext(inputfile)[0] + '.inc' - outputfile = str(outputfile).replace(sourcedir, destdir) - # print(outputfile) + outputfile = str(outputfile).replace(srcdir, destdir) + print(outputfile) if not os.path.exists(os.path.dirname(outputfile)): try: os.makedirs(os.path.dirname(outputfile)) @@ -111,7 +110,6 @@ def process_file(data): newfile.write(outfile) newfile.close() - def async_process(num): pool = mp.Pool(processes=num) pool.map(process_file, filelist) @@ -154,19 +152,25 @@ class Handler: response = dialog.run() if response == Gtk.ResponseType.OK: - cnt = sourcedir_filecnt(dialog.get_filename()) - if cnt >0: - print(cnt) + filecnt = sourcedir_filecnt(dialog.get_filename()) + if filecnt >0: + print(filecnt) app.obj("source_entry").set_text(dialog.get_filename()) app.obj("destination_label").set_sensitive(True) app.obj("destination_entry").set_sensitive(True) app.obj("destination_button").set_sensitive(True) + app.obj("numfiles_label").set_text(str(filecnt)) + app.obj("progress_label").set_text("{} of {}".format(fileindex, filecnt)) + foldercnt = sourcedir_foldercnt(dialog.get_filename()) + if foldercnt >0: + app.obj("numfolders_label").set_text(str(foldercnt)) elif response == Gtk.ResponseType.CANCEL: print("Cancel") dialog.destroy() def on_destination_button_clicked(self,widget): + global destdir dialog = Gtk.FileChooserDialog("Select destination directory!", app.obj("window"), @@ -180,11 +184,25 @@ class Handler: destdir = dialog.get_filename() app.obj("destination_entry").set_text(dialog.get_filename()) app.obj("include_checkbutton").set_sensitive(True) + app.obj("translation_frame").set_sensitive(True) + print(srcdir) + print(destdir) elif response == Gtk.ResponseType.CANCEL: print("Cancel") dialog.destroy() + def on_translate_button_clicked(self, widget): + app.obj("sourceframe").set_sensitive(False) + app.obj("translate_button").set_sensitive(False) + process_files() + + def update_pbar(self, widget): + self.val += 1/filecnt + self.pbar.set_fraction(self.val) + while Gtk.events_pending(): + Gtk.main_iteration() + class ExampleApp: global app From 4280264fa690f49abce332b1f74d254a6bbcd763 Mon Sep 17 00:00:00 2001 From: Lerking <33354709+Lerking@users.noreply.github.com> Date: Tue, 27 Feb 2018 13:18:20 +0100 Subject: [PATCH 30/37] Update Gui testing --- Gui test/gtk_test_2.py | 82 ++++ Gui test/gtk_test_3.py | 80 ++++ Gui test/gtk_test_4.py | 118 ++++++ Gui test/gtk_test_5.py | 166 ++++++++ Gui test/h2inc (copy).xml | 393 ++++++++++++++++++ Gui test/h2inc.glade~ | 367 ++++++++++++++++ Gui test/h2inc_gtk.py | 191 ++++++--- Multiprocessing test/mp_test_1.py | 18 + .../__pycache__/h2inc_gui_old.cpython-35.pyc | Bin 0 -> 6528 bytes .../__pycache__/h2inc_gui_old.cpython-36.pyc | Bin 0 -> 5649 bytes .../__pycache__/h2inc_old.cpython-36.pyc | Bin 0 -> 3468 bytes 11 files changed, 1355 insertions(+), 60 deletions(-) create mode 100644 Gui test/gtk_test_2.py create mode 100644 Gui test/gtk_test_3.py create mode 100644 Gui test/gtk_test_4.py create mode 100644 Gui test/gtk_test_5.py create mode 100644 Gui test/h2inc (copy).xml create mode 100644 Gui test/h2inc.glade~ create mode 100644 Multiprocessing test/mp_test_1.py create mode 100644 old_h2inc/__pycache__/h2inc_gui_old.cpython-35.pyc create mode 100644 old_h2inc/__pycache__/h2inc_gui_old.cpython-36.pyc create mode 100644 old_h2inc/__pycache__/h2inc_old.cpython-36.pyc diff --git a/Gui test/gtk_test_2.py b/Gui test/gtk_test_2.py new file mode 100644 index 0000000..4130fe1 --- /dev/null +++ b/Gui test/gtk_test_2.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python3.5 + +# You are free to use and/or change this code for +# your own needs. + +# Original code (c)2018 Jan Lerking +# Program to convert C-header (*.h) files to nasm include files (*.inc), +# for direct usage in assembly programming using nasm/yasm. + +import gi +gi.require_version('Gtk', '3.0') +from gi.repository import Gtk + +class FileChooserWindow(Gtk.Window): + + def __init__(self): + Gtk.Window.__init__(self, title="FileChooser Example") + + box = Gtk.Box(spacing=6) + self.add(box) + + button1 = Gtk.Button("Choose File") + button1.connect("clicked", self.on_file_clicked) + box.add(button1) + + button2 = Gtk.Button("Choose Folder") + button2.connect("clicked", self.on_folder_clicked) + box.add(button2) + + def on_file_clicked(self, widget): + dialog = Gtk.FileChooserDialog("Please choose a file", self, + Gtk.FileChooserAction.OPEN, + (Gtk.STOCK_CANCEL, Gtk.ResponseType.CANCEL, + Gtk.STOCK_OPEN, Gtk.ResponseType.OK)) + + self.add_filters(dialog) + + response = dialog.run() + if response == Gtk.ResponseType.OK: + print("Open clicked") + print("File selected: " + dialog.get_filename()) + elif response == Gtk.ResponseType.CANCEL: + print("Cancel clicked") + + dialog.destroy() + + def add_filters(self, dialog): + filter_text = Gtk.FileFilter() + filter_text.set_name("Text files") + filter_text.add_mime_type("text/plain") + dialog.add_filter(filter_text) + + filter_py = Gtk.FileFilter() + filter_py.set_name("Python files") + filter_py.add_mime_type("text/x-python") + dialog.add_filter(filter_py) + + filter_any = Gtk.FileFilter() + filter_any.set_name("Any files") + filter_any.add_pattern("*") + dialog.add_filter(filter_any) + + def on_folder_clicked(self, widget): + dialog = Gtk.FileChooserDialog("Please choose a folder", self, + Gtk.FileChooserAction.SELECT_FOLDER, + (Gtk.STOCK_CANCEL, Gtk.ResponseType.CANCEL, + "Select", Gtk.ResponseType.OK)) + dialog.set_default_size(800, 400) + + response = dialog.run() + if response == Gtk.ResponseType.OK: + print("Select clicked") + print("Folder selected: " + dialog.get_filename()) + elif response == Gtk.ResponseType.CANCEL: + print("Cancel clicked") + + dialog.destroy() + +win = FileChooserWindow() +win.connect("delete-event", Gtk.main_quit) +win.show_all() +Gtk.main() \ No newline at end of file diff --git a/Gui test/gtk_test_3.py b/Gui test/gtk_test_3.py new file mode 100644 index 0000000..40c0015 --- /dev/null +++ b/Gui test/gtk_test_3.py @@ -0,0 +1,80 @@ +import gi +gi.require_version('Gtk', '3.0') +from gi.repository import Gtk +import time +from multiprocessing import Process, Value, Lock, Pool + +class Counter(object): + def __init__(self, initval=0.0): + self.val = Value('d', initval) + self.lock = Lock() + + def increment(self): + with self.lock: + self.val.value += 1/2542 + print(self.val.value) + + def value(self): + with self.lock: + return self.val.value + +class MyWindow(Gtk.Window): + + def __init__(self): + Gtk.Window.__init__(self, title = 'Hello World') + + #self.val = 0 + + self.box = Gtk.Box(spacing=6) + self.add(self.box) + + self.button1 = Gtk.Button(label = 'Hello') + self.button1.connect('clicked', self.on_button1_clicked) + self.box.pack_start(self.button1, True, True, 0) + + self.button2 = Gtk.Button(label = 'Goodbye') + self.button2.connect('clicked', self.on_button2_clicked) + self.box.pack_start(self.button2, True, True, 0) + + self.pbar = Gtk.ProgressBar() + self.pbar.set_fraction(0.0) + self.box.pack_start(self.pbar, True, True, 0) + + def on_button1_clicked(self, widget): + print('Hello') + #procs = [Process(target=self.func, args=(counter,)) for i in range(10)] + + #for p in procs: p.start() + #for p in procs: p.join() + p = Pool(2) + p.map(self.func, range(2542)) + print (counter.value()) + + #while counter.val.value <10: + #self.update_pbar(self.pbar) + print('Goodbye') + Gtk.main_quit() + + def on_button2_clicked(self, widget): + print('Goodbye') + Gtk.main_quit() + + def update_pbar(self, widget, value): + #self.val += 0.1 + self.pbar.set_fraction(value) + #time.sleep(0.5) + while Gtk.events_pending(): + Gtk.main_iteration() + + def func(self, data): + #for i in range(2542): + time.sleep(0.01) + counter.increment() + #while counter.value() <1: + #self.update_pbar(self.pbar, counter.value()) + +counter = Counter(0.0) +win = MyWindow() +win.connect('delete-event', Gtk.main_quit) +win.show_all() +Gtk.main() diff --git a/Gui test/gtk_test_4.py b/Gui test/gtk_test_4.py new file mode 100644 index 0000000..08672c9 --- /dev/null +++ b/Gui test/gtk_test_4.py @@ -0,0 +1,118 @@ +import gobject +import pygtk +pygtk.require('2.0') +import gtk +import multiprocessing +import threading +import time + +gtk.gdk.threads_init() + +class Listener(gobject.GObject): + __gsignals__ = { + 'updated' : (gobject.SIGNAL_RUN_LAST, + gobject.TYPE_NONE, + (gobject.TYPE_FLOAT, gobject.TYPE_STRING)), + 'finished': (gobject.SIGNAL_RUN_LAST, + gobject.TYPE_NONE, + ()) + } + + def __init__(self, queue): + gobject.GObject.__init__(self) + self.queue = queue + + def go(self): + print("Listener has started") + while True: + # Listen for results on the queue and process them accordingly + data = self.queue.get() + # Check if finished + if data[1]=="finished": + print("Listener is finishing.") + self.emit("finished") + return + else: + self.emit('updated', data[0], data[1]) + +gobject.type_register(Listener) + +class Worker(): + def __init__(self, queue): + self.queue = queue + + def go(self): + print("The worker has started doing some work (counting from 0 to 9)") + for i in range(10): + proportion = (float(i+1))/10 + self.queue.put((proportion, "working...")) + time.sleep(0.5) + self.queue.put((1.0, "finished")) + print("The worker has finished.") + + +class Interface: + def __init__(self): + self.process = None + self.progress = gtk.ProgressBar() + button = gtk.Button("Go!") + button.connect("clicked", self.go) + vbox = gtk.VBox(spacing=5) + vbox.pack_start(self.progress) + vbox.pack_start(button) + vbox.show_all() + self.frame = vbox + + def main(self): + window = gtk.Window(gtk.WINDOW_TOPLEVEL) + window.set_border_width(10) + window.add(self.frame) + window.show() + window.connect("destroy", self.destroy) + + gtk.main() + + def destroy(self, widget, data=None): + gtk.main_quit() + + + def callbackDisplay(self, obj, fraction, text, data=None): + self.progress.set_fraction(fraction) + self.progress.set_text(text) + + def callbackFinished(self, obj, data=None): + if self.process==None: + raise RuntimeError("No worker process started") + print("all done; joining worker process") + self.process.join() + self.process = None + + self.progress.set_fraction(1.0) + self.progress.set_text("done") + + def go(self, widget, data=None): + if self.process!=None: + return + + print("Creating shared Queue") + queue = multiprocessing.Queue() + + print("Creating Worker") + worker = Worker(queue) + + print("Creating Listener") + listener = Listener(queue) + listener.connect("updated",self.callbackDisplay) + listener.connect("finished",self.callbackFinished) + + print("Starting Worker") + self.process = multiprocessing.Process(target=worker.go, args=()) + self.process.start() + + print("Starting Listener") + thread = threading.Thread(target=listener.go, args=()) + thread.start() + +if __name__ == '__main__': + gui = Interface() + gui.main() \ No newline at end of file diff --git a/Gui test/gtk_test_5.py b/Gui test/gtk_test_5.py new file mode 100644 index 0000000..044ed36 --- /dev/null +++ b/Gui test/gtk_test_5.py @@ -0,0 +1,166 @@ +import gi +gi.require_version('Gtk', '3.0') +from gi.repository import Gtk, GObject as gobject +import time +from multiprocessing import Process, Value, Lock, Pool, Queue +import threading + +gobject.threads_init() + +class Listener(gobject.GObject): + __gsignals__ = { + 'updated' : (gobject.SIGNAL_RUN_LAST, + gobject.TYPE_NONE, + (gobject.TYPE_FLOAT, gobject.TYPE_STRING)), + 'finished': (gobject.SIGNAL_RUN_LAST, + gobject.TYPE_NONE, + ()) + } + + def __init__(self, queue): + gobject.GObject.__init__(self) + self.queue = queue + + def go(self): + print("Listener has started") + while True: + # Listen for results on the queue and process them accordingly + data = self.queue.get() + # Check if finished + if data[1]=="finished": + print("Listener is finishing.") + self.emit("finished") + return + else: + self.emit('updated', data[0], data[1]) + +gobject.type_register(Listener) + +class Worker(): + def __init__(self, queue): + self.queue = queue + + def go(self): + print("The worker has started doing some work (counting from 0 to 9)") + for i in range(2542): + proportion = (float(i+1))/2542 + self.queue.put((proportion, "working...")) + time.sleep(0.01) + self.queue.put((1.0, "finished")) + print("The worker has finished.") + +class Counter(object): + def __init__(self, initval=0.0): + self.val = Value('d', initval) + self.lock = Lock() + + def increment(self): + with self.lock: + self.val.value += 1/2542 + print(self.val.value) + + def value(self): + with self.lock: + return self.val.value + +class MyWindow(Gtk.Window): + + def __init__(self): + Gtk.Window.__init__(self, title = 'Hello World') + + #self.val = 0 + self.process = None + + self.box = Gtk.Box(spacing=6) + self.add(self.box) + + self.button1 = Gtk.Button(label = 'Hello') + self.button1.connect('clicked', self.go) + self.box.pack_start(self.button1, True, True, 0) + + self.button2 = Gtk.Button(label = 'Goodbye') + self.button2.connect('clicked', self.on_button2_clicked) + self.box.pack_start(self.button2, True, True, 0) + + self.pbar = Gtk.ProgressBar() + self.pbar.set_fraction(0.0) + self.box.pack_start(self.pbar, True, True, 0) + + def callbackDisplay(self, obj, fraction, text, data=None): + self.pbar.set_fraction(fraction) + self.pbar.set_text(text) + + def callbackFinished(self, obj, data=None): + if self.process==None: + raise RuntimeError("No worker process started") + print("all done; joining worker process") + self.process.join() + self.process = None + + self.pbar.set_fraction(1.0) + self.pbar.set_text("done") + + print('Goodbye') + Gtk.main_quit() + + def go(self, widget, data=None): + if self.process!=None: + return + + print("Creating shared Queue") + queue = Queue() + + print("Creating Worker") + worker = Worker(queue) + + print("Creating Listener") + listener = Listener(queue) + listener.connect("updated",self.callbackDisplay) + listener.connect("finished",self.callbackFinished) + + print("Starting Worker") + self.process = Process(target=worker.go, args=()) + self.process.start() + + print("Starting Listener") + thread = threading.Thread(target=listener.go, args=()) + thread.start() + + #def on_button1_clicked(self, widget): + #print('Hello') + #procs = [Process(target=self.func, args=(counter,)) for i in range(10)] + + #for p in procs: p.start() + #for p in procs: p.join() + #p = Pool(2) + #p.map(self.func, range(2542)) + #print (counter.value()) + + #while counter.val.value <10: + #self.update_pbar(self.pbar) + #print('Goodbye') + #Gtk.main_quit() + + def on_button2_clicked(self, widget): + print('Goodbye') + Gtk.main_quit() + + #def update_pbar(self, widget, value): + #self.val += 0.1 + #self.pbar.set_fraction(value) + #time.sleep(0.5) + #while Gtk.events_pending(): + #Gtk.main_iteration() + + #def func(self, data): + #for i in range(2542): + #time.sleep(0.01) + #counter.increment() + #while counter.value() <1: + #self.update_pbar(self.pbar, counter.value()) + +#counter = Counter(0.0) +win = MyWindow() +win.connect('delete-event', Gtk.main_quit) +win.show_all() +Gtk.main() diff --git a/Gui test/h2inc (copy).xml b/Gui test/h2inc (copy).xml new file mode 100644 index 0000000..4702fd7 --- /dev/null +++ b/Gui test/h2inc (copy).xml @@ -0,0 +1,393 @@ + + + + + + False + normal + select-folder + + + False + vertical + 2 + + + False + end + + + + + + + + + False + False + 0 + + + + + + + + + + False + 5 + 5 + 5 + 5 + + + True + False + + + True + False + 0 + etched-out + + + True + False + 12 + + + True + False + + + True + False + end + 5 + 5 + 5 + True + Source: + right + + + 0 + 0 + 2 + + + + + True + True + 5 + 5 + 5 + 5 + Select source directory! + + + 2 + 0 + 2 + + + + + Source directory... + True + True + True + start + 5 + 5 + 5 + 5 + + + 4 + 0 + 2 + + + + + True + False + False + end + 5 + 5 + 5 + True + Destination: + right + start + 2.2351741811588166e-10 + + + 0 + 1 + 2 + + + + + True + False + True + 5 + 5 + 5 + 5 + Select destination directory! + + + 2 + 1 + 2 + + + + + Destination directory... + True + False + True + True + start + 5 + 5 + 5 + 5 + + + 4 + 1 + 2 + + + + + Create "include" folder if it does not exist. + True + False + True + False + 5 + 5 + 0 + True + + + 0 + 2 + 6 + + + + + + + + + True + False + Select folders + + + + + 0 + 0 + 5 + + + + + True + False + False + 5 + 5 + 0 + etched-out + + + True + False + 12 + + + True + False + + + True + False + 5 + 5 + True + 0 + etched-out + + + True + False + 12 + + + True + False + vertical + + + True + False + start + 5 + 5 + True + Number of folders: 0 + + + False + True + 0 + + + + + True + False + start + 5 + 5 + Number of files: 0 + + + False + True + 1 + + + + + + + + + True + False + Source information + + + + + 0 + 0 + + + + + Translate! + True + True + True + start + 5 + 5 + + + 0 + 1 + + + + + True + False + 5 + 5 + 5 + 0 + etched-out + + + True + False + 12 + + + True + False + vertical + + + True + False + start + 5 + 5 + Total progress: + + + False + True + 0 + + + + + True + False + 5 + 5 + 5 + + + False + True + 1 + + + + + + + + + True + False + Progress + + + + + 0 + 2 + + + + + + + + + True + False + Translation + + + + + 0 + 1 + 5 + + + + + + diff --git a/Gui test/h2inc.glade~ b/Gui test/h2inc.glade~ new file mode 100644 index 0000000..ad6a6e9 --- /dev/null +++ b/Gui test/h2inc.glade~ @@ -0,0 +1,367 @@ + + + + + + False + 5 + 5 + 5 + 5 + + + True + False + 5 + 5 + 5 + 5 + True + True + + + True + False + 0 + etched-out + + + True + False + 12 + + + True + False + + + True + False + end + 5 + 5 + 5 + True + Source: + right + + + 0 + 0 + 2 + + + + + True + True + 5 + 5 + 5 + 5 + Select source directory! + + + 2 + 0 + 2 + + + + + Source directory... + True + True + True + start + 5 + 5 + 5 + 5 + + + 4 + 0 + 2 + + + + + True + False + False + end + 5 + 5 + 5 + True + Destination: + right + start + 2.2351741811588166e-10 + + + 0 + 1 + 2 + + + + + True + False + True + 5 + 5 + 5 + 5 + Select destination directory! + + + 2 + 1 + 2 + + + + + Destination directory... + True + False + True + True + start + 5 + 5 + 5 + 5 + + + 4 + 1 + 2 + + + + + Create "include" folder if it does not exist. + True + False + True + False + 5 + 5 + 0 + True + + + 0 + 2 + 6 + + + + + + + + + True + False + Select folders + + + + + 0 + 0 + 5 + + + + + True + False + False + 5 + 5 + 0 + etched-out + + + True + False + 12 + + + True + False + + + True + False + 5 + 5 + True + 0 + etched-out + + + True + False + 12 + + + True + False + vertical + + + True + False + start + 5 + 5 + True + Number of folders: 0 + + + False + True + 0 + + + + + True + False + start + 5 + 5 + Number of files: 0 + + + False + True + 1 + + + + + + + + + True + False + Source information + + + + + 0 + 0 + + + + + Translate! + True + True + True + start + 5 + 5 + + + 0 + 1 + + + + + True + False + 5 + 5 + 5 + 0 + etched-out + + + True + False + 12 + + + True + False + vertical + + + True + False + start + 5 + 5 + Total progress: + + + False + True + 0 + + + + + True + False + 5 + 5 + 5 + + + False + True + 1 + + + + + + + + + True + False + Progress + + + + + 0 + 2 + + + + + + + + + True + False + Translation + + + + + 0 + 1 + 5 + + + + + + diff --git a/Gui test/h2inc_gtk.py b/Gui test/h2inc_gtk.py index 48bcd28..7f1c7d2 100644 --- a/Gui test/h2inc_gtk.py +++ b/Gui test/h2inc_gtk.py @@ -9,11 +9,58 @@ import os import sys -import multiprocessing as mp import io import gi gi.require_version('Gtk', '3.0') -from gi.repository import Gtk, Gio, GdkPixbuf +from gi.repository import Gtk, Gio, GObject as gobject +import time +from multiprocessing import Process, Value, Lock, Pool, Queue +import threading + +gobject.threads_init() + +class Listener(gobject.GObject): + __gsignals__ = { + 'updated' : (gobject.SIGNAL_RUN_LAST, + gobject.TYPE_NONE, + (gobject.TYPE_FLOAT, gobject.TYPE_STRING, gobject.TYPE_INT)), + 'finished': (gobject.SIGNAL_RUN_LAST, + gobject.TYPE_NONE, + ()) + } + + def __init__(self, queue): + gobject.GObject.__init__(self) + self.queue = queue + + def go(self): + print("Listener has started") + while True: + # Listen for results on the queue and process them accordingly + data = self.queue.get() + # Check if finished + if data[1]=="finished": + print("Listener is finishing.") + self.emit("finished") + return + else: + self.emit('updated', data[0], data[1], data[2]) + +gobject.type_register(Listener) + +class Worker(): + def __init__(self, queue, filecnt, filelist): + self.queue = queue + self.filecnt = filecnt + + def go(self): + print("The worker has started doing some work (counting from 0 to 9)") + for i in range(self.filecnt): + proportion = (float(i+1))/self.filecnt + self.queue.put((proportion, "working...", i)) + process_file(filelist[i]) + self.queue.put((1.0, "finished")) + print("The worker has finished.") tupline = [] preproc = () @@ -22,8 +69,9 @@ folderlist = [] cnt = 0 srcdir = '' destdir = '' -num_cores = mp.cpu_count() +#num_cores = multiprocessing.cpu_count() fileindex = 0 +filecnt = 0 def sourcedir_filecnt(sourcedir): ### Return the number of files, ending with '.h', in sourcedir - including subdirectories ### @@ -110,12 +158,66 @@ def process_file(data): newfile.write(outfile) newfile.close() -def async_process(num): - pool = mp.Pool(processes=num) - pool.map(process_file, filelist) +class ExampleApp: -class Handler: + global app + global destlabel + def __init__(self): + + self.filecnt = 0 + self.fileindex = 0 + self.process = None + self.app = Gtk.Application.new("org.h2inc", Gio.ApplicationFlags(0)) + self.app.connect("activate", self.on_app_activate) + self.app.connect("shutdown", self.on_app_shutdown) + app = self.app + + def on_app_activate(self, app): + builder = Gtk.Builder() + builder.add_from_file("h2inc.glade") + builder.connect_signals(self) + + self.obj = builder.get_object + self.obj("window").set_application(app) + self.obj("window").set_wmclass("h2inc_gtk","h2inc_gtk") + self.obj("window").show_all() + + button = Gtk.Button.new_from_stock(Gtk.STOCK_CANCEL) + button.set_property("can-default",True) + self.obj("filechooser_dialog").add_action_widget(button, Gtk.ResponseType.CANCEL) + button = Gtk.Button.new_from_stock(Gtk.STOCK_APPLY) + button.set_property("can-default",True) + self.obj("filechooser_dialog").add_action_widget(button, Gtk.ResponseType.OK) + + def on_app_shutdown(self, app): + self.app.quit() + + def run(self, argv): + self.app.run(argv) + + def callbackDisplay(self, obj, fraction, text, index, data=None): + self.obj("progress_label").set_text("{} of {}".format(index+1, self.filecnt)) + self.obj("progressbar").set_fraction(fraction) + self.obj("progressbar").set_text(text) + + def callbackFinished(self, obj, data=None): + if self.process==None: + raise RuntimeError("No worker process started") + print("all done; joining worker process") + self.process.join() + self.process = None + + self.obj("progressbar").set_fraction(1.0) + self.obj("progressbar").set_text("done") + self.obj("sourceframe").set_sensitive(True) + self.obj("translate_button").set_sensitive(True) + self.obj("destination_label").set_sensitive(True) + self.obj("destination_entry").set_sensitive(True) + self.obj("destination_button").set_sensitive(True) + self.obj("include_checkbutton").set_sensitive(True) + self.obj("translation_frame").set_sensitive(True) + def on_window_destroy(self,window): window.close() @@ -130,17 +232,6 @@ class Handler: print("File selection: %s" % widget.get_filename()) self.on_dialog_close(widget) - def on_filechooser_dialog_file_activated(self,widget): - self.on_filechooser_dialog_response(widget,-5) - - def on_filechooser_dialog_update_preview(self,widget): - if widget.get_filename() != None and os.path.isfile(widget.get_filename()): - pixbuf = GdkPixbuf.Pixbuf.new_from_file_at_scale(widget.get_filename(),200,200,True) - app.obj("preview").set_from_pixbuf(pixbuf) - - def on_file_button_clicked(self,widget): - app.obj("filechooser_dialog").show_all() - def on_source_button_clicked(self,widget): dialog = Gtk.FileChooserDialog("Select source directory!", @@ -152,15 +243,15 @@ class Handler: response = dialog.run() if response == Gtk.ResponseType.OK: - filecnt = sourcedir_filecnt(dialog.get_filename()) - if filecnt >0: - print(filecnt) + self.filecnt = sourcedir_filecnt(dialog.get_filename()) + if self.filecnt >0: + print(self.filecnt) app.obj("source_entry").set_text(dialog.get_filename()) app.obj("destination_label").set_sensitive(True) app.obj("destination_entry").set_sensitive(True) app.obj("destination_button").set_sensitive(True) - app.obj("numfiles_label").set_text(str(filecnt)) - app.obj("progress_label").set_text("{} of {}".format(fileindex, filecnt)) + app.obj("numfiles_label").set_text(str(self.filecnt)) + app.obj("progress_label").set_text("{} of {}".format(fileindex, self.filecnt)) foldercnt = sourcedir_foldercnt(dialog.get_filename()) if foldercnt >0: app.obj("numfolders_label").set_text(str(foldercnt)) @@ -192,51 +283,31 @@ class Handler: dialog.destroy() - def on_translate_button_clicked(self, widget): + def on_translate_button_clicked(self, widget, data=None): app.obj("sourceframe").set_sensitive(False) app.obj("translate_button").set_sensitive(False) - process_files() - - def update_pbar(self, widget): - self.val += 1/filecnt - self.pbar.set_fraction(self.val) - while Gtk.events_pending(): - Gtk.main_iteration() - -class ExampleApp: - - global app - global destlabel - - def __init__(self): - self.app = Gtk.Application.new("org.h2inc", Gio.ApplicationFlags(0)) - self.app.connect("activate", self.on_app_activate) - self.app.connect("shutdown", self.on_app_shutdown) - app = self.app + if self.process!=None: + return - def on_app_activate(self, app): - builder = Gtk.Builder() - builder.add_from_file("h2inc.glade") - builder.connect_signals(Handler()) + print("Creating shared Queue") + queue = Queue() - self.obj = builder.get_object - self.obj("window").set_application(app) - self.obj("window").set_wmclass("h2inc_gtk","h2inc_gtk") - self.obj("window").show_all() + print("Creating Worker") + worker = Worker(queue, self.filecnt, filelist) - button = Gtk.Button.new_from_stock(Gtk.STOCK_CANCEL) - button.set_property("can-default",True) - self.obj("filechooser_dialog").add_action_widget(button, Gtk.ResponseType.CANCEL) - button = Gtk.Button.new_from_stock(Gtk.STOCK_APPLY) - button.set_property("can-default",True) - self.obj("filechooser_dialog").add_action_widget(button, Gtk.ResponseType.OK) + print("Creating Listener") + listener = Listener(queue) + listener.connect("updated",self.callbackDisplay) + listener.connect("finished",self.callbackFinished) - def on_app_shutdown(self, app): - self.app.quit() + print("Starting Worker") + self.process = Process(target=worker.go, args=()) + self.process.start() - def run(self, argv): - self.app.run(argv) + print("Starting Listener") + thread = threading.Thread(target=listener.go, args=()) + thread.start() app = ExampleApp() app.run(sys.argv) diff --git a/Multiprocessing test/mp_test_1.py b/Multiprocessing test/mp_test_1.py new file mode 100644 index 0000000..00183ad --- /dev/null +++ b/Multiprocessing test/mp_test_1.py @@ -0,0 +1,18 @@ +from multiprocessing import Process +import os + +def info(title): + print(title) + print('module name:', __name__) + print('parent process:', os.getppid()) + print('process id:', os.getpid()) + +def f(name): + info('function f') + print('hello', name) + +if __name__ == '__main__': + info('main line') + p = Process(target=f, args=('bob',)) + p.start() + p.join() \ No newline at end of file diff --git a/old_h2inc/__pycache__/h2inc_gui_old.cpython-35.pyc b/old_h2inc/__pycache__/h2inc_gui_old.cpython-35.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ffc4e35acd846814af409637bb7a3a68b2730d84 GIT binary patch literal 6528 zcmbtYOLH7a5ze0XOVa2qTb4bN?X|~iN&5s=7~}P0jfq7|AhYYO0~4d^mNe4zw5aac zwI*y1m;f#i!Ic9Yj(}T?a6lPGx0fXJzH1GHVtlCQ5gH z_B-{rOGN*okxv@<9X#O=93mHAhdM;QL&_mPMJh#pnpB$n45AL1NR`MhlPcS}jLcO?Rmh(pH9`I)sY&vuNKILetZ-CGRmqdcToOKR4R^E_5FqH}>dbL7vHnjg$9kXn#A@GX*B6ud~CCGyXaI!FF_Qs>EE zCbdk?L$^e3nSOb~ZBI6EYm&mQ~7fg-^t`&L#Q`LlPRk++d z?bIhu*~Im%Nj*L?=_oof*^isMUdQC__M#~0nvx7Z(oNs1r%h?E-SOOZvlHx_?1OHk zPfYpVf!8|P6>e~39_qGhiaU{RclSSSYE!%)^maQQ(0ax=+s1idoE_tQR!^D2{YN_= z-u?K&eUsb%jJ;dwSgLy1mfTkUJd)J>A0Ey1_cMx{;Y3Aq5@R z(@dN=)I9K6%+wjTcUQHxjP5C5OS!!aEGGe;B=yWqD(4QB*T zjo@jDG6@e>cZ3;BTLkN-gn~uWf`><(v%9jZ48M zMT)2xE0=Mi^N9?ofy`W_-w^F#b4%o&qhG^uPKp&<7PjSak>^F^yeRhGsA8jgTTZmP zAR;a#A|!y=jDx+sKO%y?2-X#HFHppJ4$qv~ZYmLcNd#XS7yRn&vRE*Vi5;~7CI%L` zIBtQOtWq0aWz+)SI-@>V$JSXD0ab}DoL5#J@^6*gHF7VLdj%WHu(8*Kb1hj4iwpv8 zbiuPhh8IQ`Mk4!aV!6w*(B)(y*z65*N0)njeCH*;A>vGY^Xi~pc`3W> z1V{a~M2)MW#?}8{4Tw0SMr1?rDqEdbmLvz#cteJ7Bx@oLt&I_lHlT;r&B_+RI%?gs zf{V#!u8H7liQv&Nf3;w+jtRE$hOMjYyhld>r9CNyUvY7 z`??V8iPSg$=ZJuaSGxk+u&yX2Yu=DGZzOBpeCF{Q(x4Ta62DzK$z6J4e zbgj$OdLsVqL_E}ceR$jB8!ux)o3W@$?}$WH5lV5-wNwB0wnm#q{kiR)zw7B*uvgoY z8Xa!c-WsJIcuiDpp;jLUQL|G+0pHiC;9Ir+(vF8xAJuH3uc7#36zG%H{z^RKdSTS= zHludX9i85Mtef4i(~P{@y>+`pjoXH>76rBKX6V;Yp*ua-v((j7R#xS{^xFFek;z2f z)2KfgFA?i)(mMFkUIfoXN<)qo!Xf)d0cGPHW-0B3aW+%K2#B!f<|I?qqYKgD5Y9<>- za8G|}ggR!6|JDSPu?aXL5GW=eBeLnbqc1}T!Sxl!t^(Y`V|THBPkS6JYjIet#nD=8 z@73B-%?%Ja-5{!YPupR%p$dr?RX*r`(d_g*lRoi6RS5R>>@d|0R5=Mk1gFX)dN^A8 zGqK;>-Mv8j61n|S!s)G!Z94-6#q$?{jC!7NX#i2rUZialWK1ziqF&eWf?fqMxu)x4 zSKnZUPr)W;P=z1Ixp&GdpB4aAF3@eS8>!L(kbclbikgfcxSlGz9^%F4^z%-O%$xU{ zPuqUauc!1yjP*GH;46$X;cMv|y39PRuvLAXu?m1Gv^a5La8-!-d;n(h@fARo*ornm zRY7p@sxlapQ^cf0FETkB+a?>eBQVVEZlBJ=n2+> z8+Sd)fb&nYWmMT_SBI}kTFewvU@3i~CH<>|3~bojktuRW$~%0H@UnUu(kxxxG?Q`E z#;<5Lt0}^Nt%IZ8;HfH!i>;2+vJ1r3U<(SIum|tJl!vAXSpw*M0GjF`Na8gVUzy~@ zwJ(;$%yw&O*=!OOyKusU%qDNZp^{hcgKsRtILAyOITGeE3i|-$RN}v)BY&x)Q+28W z7gF=i_nd{)lJjP2*{P>);6HF@mPddX1Fk!7JF|FW-nr*22si$N!}2k-d`moEvh-ER z?N7gSPz|nU?0DYkJv`w}lmcAO_y&dh7}`a-hT*7w(0S(c(n>Gqj4 zCAYOP5-w6`bH+&1LdwxI91gf9@QX7EiFX4M6hzB6%aFxC51^j2@h%4yPpN$5`NxoJ5hRByTAlG@ge^B*bZyP4 z0qf!kIXlQ%MhaA&DI^3^06{(cE5_9Rxk1pLE%1Zj<=Q$3-albrgBNiWrKo{O!__}c z`i~M{h&35Xz>2Rq>hQC`FzWJrC9n~j1W)N@Qu$>P!u9Nn6XPTam-Kdj@&2$A7}oqa z1nQIGRs8^4(H}8*$UwaKh_N3sxW!=TIrfcp+V>f|1CSg|QVGQeLq}Z4{4eo@>~?~& z*%1Ttr}~R$%^si1tqk+y!$&^9dk;@oz=?rWLt6$jKDUW%aU>MXZE>U{eo=DhC`I}! zNwgFW1LQ4E7V!xVni!Ehgp2Ig`G)E6D>_+`*KjgfDI+gCpLMq=6XofsMEW1(JkOKU zKTprZ#ZiF{i*$t0_&eL;v(Moje8KaJJU!_;^n^pB6dwQ%)O2wMfF#@DGb$?cneYr7 z!x@p5A2fIBnQeUoM(Gze<5ucxy^#sq(3*Q9lY8*s$)laeHZm4LsTXW=ipM2Gxr0`3=c{$$W&;rjbrwhTNp1Hh z@Ab4s10uJfty-*~Dz_||8wco_kB4WDH8V597Hj&K5lxqenhwRqeB1RRcM0t0M#GdE z4K$X$4&#+ZvAy3yBaO!IIlzz;Bd)EuS zbR{h_AJx}zU>~6n;OAzxs7^D;FyQ7*dO>>*ajHujrb~=TpUN_Ab0~eGbcoU%vW0Ce zxyzWuX+8)yq*$>!rf`f~DR=&;Mkk>vigEaUB_+yM8V+VrQ_eq=v))%l+=|+9G>In3_c5dg;&BEK4ZpicQ%~kbEfZgXZbAl z@VnJ;p3kSa1-<~B*Ind`Xmj1A@H{^sUf>tPWxmX;$Ck*8!Z(&E9NYY2z>Y1x60CGC z9b3#ods!4k@z_THVz6?|cn!EKG$*`mt6ndbFwCtxtaa@dqE(na&1M_M$EH^1+M4W>56v zKw{mQIO@w5_LdDgL#wCs+!QM6ia-*wbSR@%5XXipZqe;VKO3F*@x<=|1Qut);I{ta6|@eMr4JHHY+^gqjg125~?%N&zh2A>rSv#x=ju+jI01DF#T!2GvXhjj`fd(0SC>DcJI)E^R8 z5tw49B)oLC*eMGi|J(`mtj-M?S;PsgSpj9GGlN?)QB<>PZqZUzJoAf|DB)i|an1Ub zS5{acE^#hhV-cgpWDQjn>)@)Kz!%gKQQ=`?#_V+-{qu&-SJiZf*^>+?FA$}e`@OZx zL{-dxHZf%brDp)VA%i8h{|AB)FpBL1acXl#!gRj z8S|G7*O}88{Ddz#+XZvB<*BpPKvy&Tbhf)wyVg1r?kk|G2s)u7R>UQ7`6P>btzx_i z-bF~AxTht(I^iZR=oMa>YOR*ucnvda$=+|#+m#c~=xuFUZ-(pYv?UNmtfs48!|2-o zwCc6#RTXK2c>Q#p>*!y{IN4~e-SplVs#}n_62MY@^=G`sH5K0Est$w%@$Xc{p z+ci-xzE`_DjolBLh(isuN zs@-d=!Z6fX=s8u;5L z0o<$AvvLLe@)Cg>fhz=70kD6OcKNa?+f5{$dgd%CQ4Qvjr6$+i*w}b2Q_O)fE&@O= z^sr+Ft*|ZRtx9Z!+8E~_of~)m3L2dO!LGap=6^qE-FyBiQds<49K=zlyZ3xEYW2gQ zr{d=(hpdOqSOxMqblWh91l`!{w;RB193E-^y>2ty5zYJW(wXpBgSAoq*=RJ{y|!vJ zHs0$-t!6jAk4{2ex8()UYOfWAAu{PHrKmw^bE0qv6B##F(DGnu3jcpgrkqNKj0(1< zy|iOwQD^cxPGI`-2K8AWN)jfWv!+LUu*By+NvfZ7+)_^wctfNRvX$m z3Sw^eA}HBh(;sP_dr|MpX15=3Kib_*dWDfW!_4cRu?p_N{K(uAOF~U})|>{jsiCxc zyO9iyj|}pom4Yj;*bFkhWo)L3&d6raA`5q%V<)kljLxZ%U&(da@hX6Jn?m5y|3yll zfy_vW`@c%P`;>M(^#OQ(Ka%Z1)PwW$f(T$vA-Qx=x4DZM7@N+7kqAzyK!I7@Atok8x zWdaofBqCl&oU==Htu*pVF_4Qqk`#=1W$eI>q?FY;GF`(%A&Rz@GoVcHhEtbQ>rPoD zNmu8f$kzVBPV|E3jjoe3&X`-k;>K6tpoBc=2D&g2O-w5QPq9{4#@3SV!KuDA!98Kj z8!;zWqtMC=BlFmS5=v%v!~umG|NANEUNJTDMeUCQ$vj5!0Kj5?`gfW6+b%1zqJb5= z%&xMEU11CMEL*ZKphcgRX#|Kd;1c!0QD)a!#ZaRKMM6g~p{vk*KEYS9?qKG%@HVcq zN%3^khj`+D04S=IZRK=OS06K#ZMZ7iapm_z*72xksw+R~y|jm@_zyDjdts~G&^~61 z)=Sj><9?X47)PveqNflqZ$nII zC`=BfRvXt7^#yH5F(q2Huf66DqN3WD9>{gE86S)n(-|2>3NA1zBjis&Bgtamx=QNo zAzu5GcJ=}#DX`0*5tt=#pTN%vJRq=1;Ku|O2z&rgcN6n0r-GtVVw z#+zP}MkvQ7xnvE!9-f%kEjA02EwVfe7$yrNoc!GsJ2*c!va|VoY;2S<$Ht~GAY(hW zl-+QY-Erh0OeI4-V`NY(#_rg3ypFGO!cGk$|H`QF%Bc9ts5BWFLvXeRi<^_IJ;}}! z<(7@M2ZkeT^q3by^T2 z33x14d$4#`oN07yQ4ppdgmw0m;REMM-zmq#~@Kn8Q!xydD;nYW=uHA2`M6|!q6JJm~qlNJ+mb@aF>C9CoNmvuB&3A z2BVM0l@>`^lC!{z~f=m6@I?MG5ZQc_`|-A9j}J=uPmXmt^ox}P|V z(U$QV%Eb6~5#cC0L-I!{ZOUft1TT{MVygVQ3+g}OiK&#HM0+1myUJ`t=?XMlg{I4F zFgw=nI3=~H=apg%)d+>b9r*i$#N;(Ry{{KmNeXG=zODrf~?=AezDQ`O~2VqX5=HRC)Ws&!%5PS zd`5s1nE2$xF_$nwMW=aNLCXHZulhxQ#&;ymd{|%Ao;pBS#+Nk`r*sIAQ<~DpR8mRw zAO|%?j3gx{RUVafNohs$OzJCBT2V2SM0R7!#zMU)-Y}_yj-&lUygpLljNEV(^PCY5 zz0;V4k;Hv6V{=RswX~b4p6BrT+8)p=98bQq;cbY+FWS9s6dk@v3V2CM1bUxt5w8MV agzZ*g%Vfor2B?~;+U4I_vo7iJqr?$D%hIo$Mp2fbkE~Mb!jR4@6TVh|GvuD zf7rs$K>jfnd4Ux$-~-lG>hx9h^r>wB_9gfcS6mvK+_)B6*VEF`# z>|zCc$}heSS&v8Titn-)`$y)~n3_}TVmGo-emG4;iJo!AdL|fGT;wATW|K{MXhjBC zTP#X!u?}xC+X@qN%@ybDEPuexRu5Q2b4lz8n#9D|?tVwi_xwR2sf<7>>)amnRbmjF zyF#U1ALO|gc2)Y+8@K~chAJ@Q0%q3eh8s9W)Tr#Q4_=!zqBd8`$&`WWTy z(WP3u8T**r;_z>yTZxHh%&?42p5@Ct&DZ&xhPaKoS3+)~LWleanTxL$!u=_Rn^V9e zLzFHyA_GI7n_8g}S=S*14#qbuGIudSlnjvzO)$5ZZDI;|9Qw1|0SiqGv$c0KFzrDa z_$e0oK2{wTu@{?D9`PQ-eBwUDe4>lL8fO-E<&>L(AyQ~6(pVJ?Wo(xOEm6lXuGUff zKKqa|;{?OeqX#HU8Y{^r_ZNbsQ-fjJemDpmdmtYnbKOUnXkSPO+KPhqq`LiodWsyi z%#uuf3rTT{dMr`3Ow|fmWNuQg5}`XBoxV0(v58jqu}GRbvo^Vf*~($QES@%&P2KL$ zl;}qxBVOn0Lu4Xuv&gV{X!J1Rz+ZCAmf1^$$uM(&{aa$2rR)e ztiUR)!8+WA4R`}KVGG`bx8QAf2i}GE;C=W2^6&$=13!ch;V#^R`|u+uz>nc4@DUVY z8y-Lj%20tS)SwOxXhI8i;A8kHJcOUYBltNyhF`!h;S<<}C-5uyHGB%6!EfNV@H_Y% z_TUTHhu^~=;3<3w&)_RKfJ1l=ZTKS`!3+2rj^R&W!wEP5;C5hYUL4X%d9gpWF7L(X z3{E2pIxo%Zi;Uxyg_l{E&es9+?^rCf`q^)}xW^fNVFJ9!+Qh@b*>iTL4%t4|2kc@q zN}%WEX)@wbatp7(Z~0jbueU@mc}4wFy>u@V;Ux*(w;1%$A{Az%)D`dD3UlGom4PzZ zEbDrOTCUKC{MY3sYE~DudS*48*_xi&KRKJGIh$-o>E3#{-n-p5alZff3(o&JJuBGy z=ky`-jYAglC&?V7)H`*sC+M_J665f+*qWpUcIf%V@+5ce`4F7T zV!2W;y&_dq!j_KbDeJ`R)K!M+ZeKch(K@42wMnGXtP-hJYMR8#dZ|QGvl5e7*=*KH z*{W-0tB&4DnkgHxq22jYq?yD+c6YcuGGsjTA9)6jMio}?pLYDek%3mOy5!|XVkz* zlqX4d>=b*WNlsHw7AvJv?MAvNRm#=arY}^ob5g06D|iu{^i`^JQf`&1C1rFcldLC$ zN~u!Op5{Dda6zg}Ru&~a(1M~mSzeIbi*W9?FD4mmYqjb-#mbzm&e__WtZ@R;LX(6vt{$$sFqieg5?86B%+e^M^hT9e$+c8boW z@CF0VnPf=n2-odTZV~G_?!fk8k|jYO0$eTeA{g3!Y!i+}-;-UH8ilx7>bP*(^-jZL z&y$no8#d14T%Ki;&Q}$E#zV;0k!)^ekWt@>UYR4>0aRTE%7<@gimNKJ+%9T<52a*Q|(+Oa0Py#l24u-e)jZB zl{gpZvyjEt&kk*s3NRLkyHv*WN7wB2lBD9UGETb)BJ~QgXnbvCx)+X0`R+OOe(m`b zd7Rv1EOH+!hEFB|gJt|y^+#Mf1oW4run{NFl$TPi49A1eo7vLt(jyX7RZAKPA0rwe9IR5| z7*p-Lcpxyz2zK;D#1@)ru(pu$j58V|%o+QkAnd6zu?AWilf Date: Tue, 27 Feb 2018 20:25:00 +0100 Subject: [PATCH 31/37] Updated h2inc_gtk.py --- Gui test/h2inc_gtk.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/Gui test/h2inc_gtk.py b/Gui test/h2inc_gtk.py index 7f1c7d2..c6bf56b 100644 --- a/Gui test/h2inc_gtk.py +++ b/Gui test/h2inc_gtk.py @@ -58,6 +58,7 @@ class Worker(): for i in range(self.filecnt): proportion = (float(i+1))/self.filecnt self.queue.put((proportion, "working...", i)) + time.sleep(0.01) process_file(filelist[i]) self.queue.put((1.0, "finished")) print("The worker has finished.") @@ -106,12 +107,6 @@ def sourcedir_foldercnt(sourcedir): # print(len(folderlist)) return cnt -def process_files(): - print(srcdir) - print(destdir) - pool = mp.Pool(processes=num_cores) - pool.map(process_file, filelist) - def process_file(data): outfile = '' inputfile = data @@ -301,13 +296,15 @@ class ExampleApp: listener.connect("updated",self.callbackDisplay) listener.connect("finished",self.callbackFinished) - print("Starting Worker") - self.process = Process(target=worker.go, args=()) - self.process.start() - print("Starting Listener") thread = threading.Thread(target=listener.go, args=()) thread.start() + print("Starting Worker") + self.process = Process(target=worker.go, args=()) + self.process.start() + + + app = ExampleApp() app.run(sys.argv) From a098969f3eadcf505fbc621132e9df64082d2868 Mon Sep 17 00:00:00 2001 From: Jan Lerking Date: Tue, 27 Feb 2018 21:19:29 +0100 Subject: [PATCH 32/37] Updated h2inc_gtk.py, h2inc.glade --- Gui test/h2inc (copy).xml | 393 --------------------------------- Gui test/h2inc.glade | 2 +- Gui test/h2inc.glade~ | 213 +++++++++++++----- Gui test/h2inc_gtk.py | 41 ++-- Gui test/test.glade~ | 446 -------------------------------------- 5 files changed, 177 insertions(+), 918 deletions(-) delete mode 100644 Gui test/h2inc (copy).xml delete mode 100644 Gui test/test.glade~ diff --git a/Gui test/h2inc (copy).xml b/Gui test/h2inc (copy).xml deleted file mode 100644 index 4702fd7..0000000 --- a/Gui test/h2inc (copy).xml +++ /dev/null @@ -1,393 +0,0 @@ - - - - - - False - normal - select-folder - - - False - vertical - 2 - - - False - end - - - - - - - - - False - False - 0 - - - - - - - - - - False - 5 - 5 - 5 - 5 - - - True - False - - - True - False - 0 - etched-out - - - True - False - 12 - - - True - False - - - True - False - end - 5 - 5 - 5 - True - Source: - right - - - 0 - 0 - 2 - - - - - True - True - 5 - 5 - 5 - 5 - Select source directory! - - - 2 - 0 - 2 - - - - - Source directory... - True - True - True - start - 5 - 5 - 5 - 5 - - - 4 - 0 - 2 - - - - - True - False - False - end - 5 - 5 - 5 - True - Destination: - right - start - 2.2351741811588166e-10 - - - 0 - 1 - 2 - - - - - True - False - True - 5 - 5 - 5 - 5 - Select destination directory! - - - 2 - 1 - 2 - - - - - Destination directory... - True - False - True - True - start - 5 - 5 - 5 - 5 - - - 4 - 1 - 2 - - - - - Create "include" folder if it does not exist. - True - False - True - False - 5 - 5 - 0 - True - - - 0 - 2 - 6 - - - - - - - - - True - False - Select folders - - - - - 0 - 0 - 5 - - - - - True - False - False - 5 - 5 - 0 - etched-out - - - True - False - 12 - - - True - False - - - True - False - 5 - 5 - True - 0 - etched-out - - - True - False - 12 - - - True - False - vertical - - - True - False - start - 5 - 5 - True - Number of folders: 0 - - - False - True - 0 - - - - - True - False - start - 5 - 5 - Number of files: 0 - - - False - True - 1 - - - - - - - - - True - False - Source information - - - - - 0 - 0 - - - - - Translate! - True - True - True - start - 5 - 5 - - - 0 - 1 - - - - - True - False - 5 - 5 - 5 - 0 - etched-out - - - True - False - 12 - - - True - False - vertical - - - True - False - start - 5 - 5 - Total progress: - - - False - True - 0 - - - - - True - False - 5 - 5 - 5 - - - False - True - 1 - - - - - - - - - True - False - Progress - - - - - 0 - 2 - - - - - - - - - True - False - Translation - - - - - 0 - 1 - 5 - - - - - - diff --git a/Gui test/h2inc.glade b/Gui test/h2inc.glade index 128c53f..d6b0cad 100644 --- a/Gui test/h2inc.glade +++ b/Gui test/h2inc.glade @@ -182,8 +182,8 @@ 5 5 0 - True True + 0 diff --git a/Gui test/h2inc.glade~ b/Gui test/h2inc.glade~ index ad6a6e9..65decf7 100644 --- a/Gui test/h2inc.glade~ +++ b/Gui test/h2inc.glade~ @@ -2,26 +2,54 @@ - + + False + dialog + + + False + vertical + 2 + + + False + + + + + + + + + + + + False + False + 0 + + + + + + + + + False - 5 - 5 - 5 - 5 - + True False - 5 - 5 - 5 - 5 - True - True + vertical True False + 5 + 5 + 5 + 5 0 etched-out @@ -34,10 +62,11 @@ True False - + True False end + 5 5 5 5 @@ -52,11 +81,9 @@ - + True True - 5 - 5 5 5 Select source directory! @@ -68,8 +95,8 @@ - - Source directory... + + Select source directory... True True True @@ -78,6 +105,7 @@ 5 5 5 + 4 @@ -86,11 +114,12 @@ - + True False False end + 5 5 5 5 @@ -107,12 +136,10 @@ - + True False True - 5 - 5 5 5 Select destination directory! @@ -124,8 +151,8 @@ - - Destination directory... + + Select destination directory... True False True @@ -135,6 +162,7 @@ 5 5 5 + 4 @@ -143,16 +171,20 @@ - + Create "include" folder if it does not exist. True False True False + 5 + 5 5 5 0 + True True + 0 @@ -168,21 +200,25 @@ True False + 5 + 5 Select folders - 0 - 0 - 5 + False + True + 0 - + True False False + 5 + 5 5 5 0 @@ -201,6 +237,7 @@ True False 5 + 5 5 True 0 @@ -211,19 +248,38 @@ False 12 - + True False vertical - + True False - start - 5 - 5 - True - Number of folders: 0 + + + True + False + 5 + 5 + Number of folders: + + + 0 + 0 + + + + + True + False + 0 + + + 1 + 0 + + False @@ -232,13 +288,33 @@ - + True False - start - 5 - 5 - Number of files: 0 + + + True + False + 5 + 5 + Number of files: + + + 0 + 0 + + + + + True + False + 0 + + + 1 + 0 + + False @@ -254,6 +330,8 @@ True False + 5 + 5 Source information @@ -264,7 +342,7 @@ - + Translate! True True @@ -272,6 +350,7 @@ start 5 5 + 0 @@ -293,37 +372,49 @@ False 12 - + True False - vertical + True - + True False start - 5 - 5 - Total progress: + False + Total progress: - False - True - 0 + 0 + 0 - + + True + False + start + True + 0 of 0 + + + 1 + 0 + + + + True False 5 5 5 + True - False - True - 1 + 0 + 1 + 2 @@ -331,9 +422,11 @@ - + True False + 5 + 5 Progress @@ -351,14 +444,16 @@ True False + 5 + 5 Translation - 0 - 1 - 5 + False + True + 1 diff --git a/Gui test/h2inc_gtk.py b/Gui test/h2inc_gtk.py index c6bf56b..43b5be5 100644 --- a/Gui test/h2inc_gtk.py +++ b/Gui test/h2inc_gtk.py @@ -73,6 +73,7 @@ destdir = '' #num_cores = multiprocessing.cpu_count() fileindex = 0 filecnt = 0 +incinc = False def sourcedir_filecnt(sourcedir): ### Return the number of files, ending with '.h', in sourcedir - including subdirectories ### @@ -159,7 +160,7 @@ class ExampleApp: global destlabel def __init__(self): - + self.filecnt = 0 self.fileindex = 0 self.process = None @@ -169,6 +170,7 @@ class ExampleApp: app = self.app def on_app_activate(self, app): + builder = Gtk.Builder() builder.add_from_file("h2inc.glade") builder.connect_signals(self) @@ -177,13 +179,12 @@ class ExampleApp: self.obj("window").set_application(app) self.obj("window").set_wmclass("h2inc_gtk","h2inc_gtk") self.obj("window").show_all() + self.obj("include_checkbutton").set_active(True) button = Gtk.Button.new_from_stock(Gtk.STOCK_CANCEL) button.set_property("can-default",True) - self.obj("filechooser_dialog").add_action_widget(button, Gtk.ResponseType.CANCEL) button = Gtk.Button.new_from_stock(Gtk.STOCK_APPLY) button.set_property("can-default",True) - self.obj("filechooser_dialog").add_action_widget(button, Gtk.ResponseType.OK) def on_app_shutdown(self, app): self.app.quit() @@ -194,7 +195,6 @@ class ExampleApp: def callbackDisplay(self, obj, fraction, text, index, data=None): self.obj("progress_label").set_text("{} of {}".format(index+1, self.filecnt)) self.obj("progressbar").set_fraction(fraction) - self.obj("progressbar").set_text(text) def callbackFinished(self, obj, data=None): if self.process==None: @@ -204,7 +204,6 @@ class ExampleApp: self.process = None self.obj("progressbar").set_fraction(1.0) - self.obj("progressbar").set_text("done") self.obj("sourceframe").set_sensitive(True) self.obj("translate_button").set_sensitive(True) self.obj("destination_label").set_sensitive(True) @@ -230,7 +229,7 @@ class ExampleApp: def on_source_button_clicked(self,widget): dialog = Gtk.FileChooserDialog("Select source directory!", - app.obj("window"), + self.obj("window"), Gtk.FileChooserAction.SELECT_FOLDER, (Gtk.STOCK_CANCEL, Gtk.ResponseType.CANCEL, Gtk.STOCK_APPLY, Gtk.ResponseType.OK)) @@ -241,15 +240,15 @@ class ExampleApp: self.filecnt = sourcedir_filecnt(dialog.get_filename()) if self.filecnt >0: print(self.filecnt) - app.obj("source_entry").set_text(dialog.get_filename()) - app.obj("destination_label").set_sensitive(True) - app.obj("destination_entry").set_sensitive(True) - app.obj("destination_button").set_sensitive(True) - app.obj("numfiles_label").set_text(str(self.filecnt)) - app.obj("progress_label").set_text("{} of {}".format(fileindex, self.filecnt)) + self.obj("source_entry").set_text(dialog.get_filename()) + self.obj("destination_label").set_sensitive(True) + self.obj("destination_entry").set_sensitive(True) + self.obj("destination_button").set_sensitive(True) + self.obj("numfiles_label").set_text(str(self.filecnt)) + self.obj("progress_label").set_text("{} of {}".format(fileindex, self.filecnt)) foldercnt = sourcedir_foldercnt(dialog.get_filename()) if foldercnt >0: - app.obj("numfolders_label").set_text(str(foldercnt)) + self.obj("numfolders_label").set_text(str(foldercnt)) elif response == Gtk.ResponseType.CANCEL: print("Cancel") @@ -259,7 +258,7 @@ class ExampleApp: global destdir dialog = Gtk.FileChooserDialog("Select destination directory!", - app.obj("window"), + self.obj("window"), Gtk.FileChooserAction.SELECT_FOLDER, (Gtk.STOCK_CANCEL, Gtk.ResponseType.CANCEL, Gtk.STOCK_APPLY, Gtk.ResponseType.OK)) @@ -268,9 +267,9 @@ class ExampleApp: response = dialog.run() if response == Gtk.ResponseType.OK: destdir = dialog.get_filename() - app.obj("destination_entry").set_text(dialog.get_filename()) - app.obj("include_checkbutton").set_sensitive(True) - app.obj("translation_frame").set_sensitive(True) + self.obj("destination_entry").set_text(dialog.get_filename()) + self.obj("include_checkbutton").set_sensitive(True) + self.obj("translation_frame").set_sensitive(True) print(srcdir) print(destdir) elif response == Gtk.ResponseType.CANCEL: @@ -278,9 +277,13 @@ class ExampleApp: dialog.destroy() + def on_include_checkbutton_toggled(self, widget): + global incinc + incinc = self.obj("include_checkbutton").get_active() + def on_translate_button_clicked(self, widget, data=None): - app.obj("sourceframe").set_sensitive(False) - app.obj("translate_button").set_sensitive(False) + self.obj("sourceframe").set_sensitive(False) + self.obj("translate_button").set_sensitive(False) if self.process!=None: return diff --git a/Gui test/test.glade~ b/Gui test/test.glade~ deleted file mode 100644 index a48e0df..0000000 --- a/Gui test/test.glade~ +++ /dev/null @@ -1,446 +0,0 @@ - - - - - - False - dialog - select-folder - - - False - vertical - 2 - - - False - end - - - gtk-ok - True - True - True - True - True - - - True - True - 0 - - - - - gtk-cancel - True - True - True - True - True - - - True - True - 1 - - - - - False - False - 0 - - - - - - - - - button4 - button5 - - - - False - - - True - False - vertical - - - True - False - 5 - 5 - 5 - 0 - etched-out - - - True - False - 12 - - - True - False - - - True - False - end - 5 - 5 - 5 - Source: - start - - - 0 - 0 - - - - - True - False - False - end - 5 - 5 - 5 - Destination: - 0.059999999999999998 - - - 0 - 1 - - - - - True - True - 5 - 5 - - - 1 - 0 - - - - - True - False - True - 5 - 5 - - - 1 - 1 - - - - - Select source directory... - True - True - True - start - 5 - 5 - 5 - 0.52999997138977051 - - - - 2 - 0 - - - - - Select destination directory... - True - False - True - True - start - 5 - 10 - 5 - 5 - - - - 2 - 1 - - - - - checkbutton - True - False - True - False - 0 - 0.47999998927116394 - True - True - - - 0 - 2 - 3 - - - - - - - - - True - False - Select folders - - - - - True - True - 0 - - - - - True - False - False - 5 - 5 - 5 - 0 - etched-out - - - True - False - 12 - - - True - False - vertical - - - True - False - 10 - 5 - 0 - etched-out - - - True - False - 12 - - - True - False - - - True - False - end - 5 - Number of folders: - - - 0 - 0 - - - - - True - False - 0.98999999999999999 - start - 5 - 0 - end - - - 1 - 0 - - - - - True - False - end - 5 - 5 - Number of files: - - - 0 - 1 - - - - - True - False - start - 5 - 5 - 0 - - - 1 - 1 - - - - - - - - - True - False - Source information - - - - - False - True - 0 - - - - - Translate! - True - True - True - start - 5 - 5 - - - False - False - 1 - - - - - True - False - 10 - 5 - 0 - etched-out - - - True - False - 12 - - - True - False - - - True - False - end - 5 - 5 - Total progress: - - - 0 - 0 - - - - - True - False - start - 5 - 5 - True - 0 of 0 - - - 1 - 0 - - - - - True - False - 10 - 5 - - - 0 - 1 - 2 - - - - - - - - - True - False - Progress - - - - - False - True - 2 - - - - - - - - - True - False - Translation! - - - - - True - True - 1 - - - - - - From 53b6172eb011726767546c423656d96ef441197f Mon Sep 17 00:00:00 2001 From: Jan Lerking Date: Tue, 27 Feb 2018 21:36:47 +0100 Subject: [PATCH 33/37] Added default dir checkbutton --- Gui test/h2inc.glade | 24 ++++++++++++++++++++++-- Gui test/h2inc.glade~ | 21 ++++++++++++++++++--- Gui test/h2inc_gtk.py | 6 ++++++ 3 files changed, 46 insertions(+), 5 deletions(-) diff --git a/Gui test/h2inc.glade b/Gui test/h2inc.glade index d6b0cad..df693c9 100644 --- a/Gui test/h2inc.glade +++ b/Gui test/h2inc.glade @@ -86,7 +86,7 @@ True 5 5 - Select source directory! + /usr/include 2 @@ -142,7 +142,7 @@ True 5 5 - Select destination directory! + ~/include 2 @@ -185,6 +185,26 @@ True + + 0 + 3 + 6 + + + + + Use default directories! + True + True + False + 5 + 5 + 5 + 5 + 0 + True + + 0 2 diff --git a/Gui test/h2inc.glade~ b/Gui test/h2inc.glade~ index 65decf7..b04c077 100644 --- a/Gui test/h2inc.glade~ +++ b/Gui test/h2inc.glade~ @@ -86,7 +86,7 @@ True 5 5 - Select source directory! + /usr/include 2 @@ -142,7 +142,7 @@ True 5 5 - Select destination directory! + ~/include 2 @@ -182,10 +182,25 @@ 5 5 0 - True True + + 0 + 3 + 6 + + + + + Use default directories! + True + True + False + 0 + True + + 0 2 diff --git a/Gui test/h2inc_gtk.py b/Gui test/h2inc_gtk.py index 43b5be5..a268040 100644 --- a/Gui test/h2inc_gtk.py +++ b/Gui test/h2inc_gtk.py @@ -74,6 +74,7 @@ destdir = '' fileindex = 0 filecnt = 0 incinc = False +defdir = False def sourcedir_filecnt(sourcedir): ### Return the number of files, ending with '.h', in sourcedir - including subdirectories ### @@ -179,6 +180,7 @@ class ExampleApp: self.obj("window").set_application(app) self.obj("window").set_wmclass("h2inc_gtk","h2inc_gtk") self.obj("window").show_all() + self.obj("default_dir_checkbutton").set_active(False) self.obj("include_checkbutton").set_active(True) button = Gtk.Button.new_from_stock(Gtk.STOCK_CANCEL) @@ -281,6 +283,10 @@ class ExampleApp: global incinc incinc = self.obj("include_checkbutton").get_active() + def on_default_dir_checkbutton_toggled(self, widget): + global defdir + defdir = self.obj("default_dir_checkbutton").get_active() + def on_translate_button_clicked(self, widget, data=None): self.obj("sourceframe").set_sensitive(False) self.obj("translate_button").set_sensitive(False) From 7b369156d4eca8e7b0724489357c1ad85580002e Mon Sep 17 00:00:00 2001 From: Jan Lerking Date: Tue, 27 Feb 2018 21:43:30 +0100 Subject: [PATCH 34/37] Updated h2inc.glade --- Gui test/h2inc.glade~ | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Gui test/h2inc.glade~ b/Gui test/h2inc.glade~ index b04c077..df693c9 100644 --- a/Gui test/h2inc.glade~ +++ b/Gui test/h2inc.glade~ @@ -197,6 +197,10 @@ True True False + 5 + 5 + 5 + 5 0 True From 825e41c3a8b6acd7acb70ab5e844720043022ec9 Mon Sep 17 00:00:00 2001 From: Jan Lerking Date: Sun, 4 Mar 2018 07:56:14 +0100 Subject: [PATCH 35/37] Updated mp tests --- Multiprocessing test/mp_test_2.py | 21 ++ Multiprocessing test/mp_test_3.py | 361 ++++++++++++++++++++++++++++++ 2 files changed, 382 insertions(+) create mode 100644 Multiprocessing test/mp_test_2.py create mode 100644 Multiprocessing test/mp_test_3.py diff --git a/Multiprocessing test/mp_test_2.py b/Multiprocessing test/mp_test_2.py new file mode 100644 index 0000000..b94505d --- /dev/null +++ b/Multiprocessing test/mp_test_2.py @@ -0,0 +1,21 @@ +import multiprocessing +import time + +data = (['a', '2'], ['b', '4'], ['c', '6'], ['d', '8'], + ['e', '1'], ['f', '3'], ['g', '5'], ['h', '7'] +) + + +def mp_handler(var1): + for indata in var1: + p = multiprocessing.Process(target=mp_worker, args=(indata[0], indata[1])) + p.start() + + +def mp_worker(inputs, the_time): + print " Processs %s\tWaiting %s seconds" % (inputs, the_time) + time.sleep(int(the_time)) + print " Process %s\tDONE" % inputs + +if __name__ == '__main__': + mp_handler(data) \ No newline at end of file diff --git a/Multiprocessing test/mp_test_3.py b/Multiprocessing test/mp_test_3.py new file mode 100644 index 0000000..d9a6d62 --- /dev/null +++ b/Multiprocessing test/mp_test_3.py @@ -0,0 +1,361 @@ +#!/usr/bin/env python + +import sys #exit +import gobject +import pygtk +pygtk.require('2.0') +import gtk +import multiprocessing +import threading +import time + +gtk.gdk.threads_init() + + +# Any time you connect to any of these signals, they should be +# protected with a threading.Lock or similar. +class ObserverBase(gobject.GObject): + __gsignals__ = { + 'message_event' : (gobject.SIGNAL_RUN_LAST, + gobject.TYPE_NONE, + (gobject.TYPE_STRING,)), + 'updated' : (gobject.SIGNAL_RUN_LAST, + gobject.TYPE_NONE, + (gobject.TYPE_FLOAT,)), + 'finished': (gobject.SIGNAL_RUN_LAST, + gobject.TYPE_NONE, + ()), + 'errored': (gobject.SIGNAL_RUN_LAST, + gobject.TYPE_NONE, + ()) + } + + def __init__(self): + gobject.GObject.__init__(self) + +gobject.type_register(ObserverBase) + +class Observer(ObserverBase): + def __init__(self, queue): + ObserverBase.__init__(self) + self.queue = queue + + def start(self): + # Create new thread to do the listening + self.thread = threading.Thread(target=self.listen, args=()) + self.thread.start() + + def join(self): + self.thread.join() + + def listen(self): + while True: + # Listen for results on the queue and process them accordingly + print "Observer is waiting for data" + data = self.queue.get() + print "Observer received: ",data + dataType = type(data).__name__ + if dataType=="bool" and data==True: + self.emit('finished') + return + if dataType=="bool" and data==False: + self.emit('errored') + return + elif dataType=="str": + self.emit('message_event', data) + elif dataType=="float": + self.emit('updated', data) + else: + raise RuntimeError("Queue had an unknown data type ('"+dataType+"') with value '"+str(data)+"'") + +gobject.type_register(Observer) + + +class Process(gobject.GObject): + __gsignals__ = { + 'starting': (gobject.SIGNAL_RUN_LAST, + gobject.TYPE_NONE, + ()), + 'joined': (gobject.SIGNAL_RUN_LAST, + gobject.TYPE_NONE, + ()) + } + + def __init__(self, function, arguments): + gobject.GObject.__init__(self) + # Create queue + self.queue = multiprocessing.Queue() + + # Create process + if arguments==(): arguments=(self.queue,) + self.process = multiprocessing.Process(target=function, args=arguments) + + # Create observer + self.observer = Observer(self.queue) + + + def getObserver(self): + return self.observer + + def start(self): + # Starting observer + self.emit("starting") + print "starting observer" + self.observer.start() + # Start process + print "starting process" + self.process.start() + + def join(self): + # Clean up after process + print "cleaning up after process" + self.process.join() + self.emit("joined") + # Clean up after observer + print "cleaning up after observer" + self.queue.put(False) # If the false is ever read, it's an error + self.observer.join() + # Clean up queue + #self.queue.close() + + +gobject.type_register(Process) + + + + +class CombinedObserver(ObserverBase): + def __init__(self, observers, weights=None): + ObserverBase.__init__(self) + self.observers = observers + numObservers = len(observers) + if weights==None: + self.weights = [1 for i in range(numObservers)] + else: + self.weights = weights + + sum_weights = sum(self.weights) + self.weights = [float(w)/sum_weights for w in self.weights] + print "weights = ",self.weights + + index = 0 + for observer in self.observers: + #observer.connect("message_event", onMessageReceived, index) + observer.connect("updated", self.onUpdate, index) + observer.connect("finished", self.onFinished, index) + observer.connect("errored", self.onErrored, index) + index = index+1 + + self.fractions = [0 for i in range(numObservers)] + self.finished = [False for i in range(numObservers)] + self.errored = [False for i in range(numObservers)] + self.lock = threading.RLock() + + def onUpdate(self, widget, fraction, index): + self.lock.acquire() + self.fractions[index] = fraction * self.weights[index] + self.emit("updated", sum(self.fractions)) + self.lock.release() + + def onFinished(self, widget, index): + self.lock.acquire() + self.finished[index] = True + if all(self.finished): + self.emit("finished") + self.lock.release() + + def onErrored(self, widget, index): + self.lock.acquire() + self.errored[index] = True + self.emit("errored") + self.lock.release() + + + +gobject.type_register(CombinedObserver) + + +# A ProgressWorker is responsible for (a) getting the job done and (b) +# reporting progress. Progress is reported via the queue. Fractions +# (from 0 to 1) indicate the proportion of the job that is finished. +# Strings may indicate what has been done or what is about to be done. +# The boolean 'True' is placed on the queue when the job is finished. +class ProgressWorker(): + def getResult(self): + return None + + def setQueue(self, queue): + self.queue = queue + + def go(self): + raise RuntimeError("Worker.go() has not been overridden") + + +class WorkerExample(ProgressWorker): + def __init__(self): + #Worker.__init__(self) + manager = multiprocessing.Manager() + self.result = manager.list() + + def getResult(self): + return self.result + + def go(self, queue): + self.setQueue(queue) + print "The worker has started doing some work (counting from 0 to 9)" + for i in range(10): + self.queue.put("working on step "+str(i)+"...") + proportion = (float(i)+1)/10 + time.sleep(0.75) + self.queue.put(proportion) + self.queue.put("done.") + self.queue.put(1.0) + self.queue.put(True) + print "The worker has finished." + self.result.append("The the work has been finished and the result isn't 42.") + + + + +class ProgressBar(gobject.GObject): + __gsignals__ = { + 'updated' : (gobject.SIGNAL_RUN_LAST, + gobject.TYPE_NONE, + (gobject.TYPE_FLOAT,)), + 'finished': (gobject.SIGNAL_RUN_LAST, + gobject.TYPE_NONE, + ()) + } + + def __init__(self, observer): + gobject.GObject.__init__(self) + self.observer = observer + self.observer.connect("updated",self.displayUpdate) + self.observer.connect("message_event",self.displayMessage) + self.observer.connect("finished",self.finish) + self.progress = gtk.ProgressBar() + self.progress.connect("destroy", self.destroy) + self.process=None + self.lock = threading.RLock() + + def displayUpdate(self, obj, fraction): + self.lock.acquire() + self.progress.set_fraction(fraction) + self.emit("updated", fraction) + self.lock.release() + + def displayMessage(self, obj, text): + self.lock.acquire() + self.progress.set_text(text) + self.lock.release() + + def finish(self, obj, data=None): + self.emit("finished") + + def destroy(self, widget, data=None): + print "Destroying" + if self.process==None: + return + self.queue.put("finished") + self.process.join() + self.thread.join() + + + +gobject.type_register(ProgressBar) + + +class ProgressBarWindow: + def __init__(self, observer, title): + self.progressBar = ProgressBar(observer) + self.progressBar.connect("finished", self.done) + window = gtk.Window(gtk.WINDOW_TOPLEVEL) + window.set_border_width(10) + window.set_title(title) + window.add(self.progressBar.progress) + window.show_all() + window.connect("destroy", self.destroy) + + def main(self): + gtk.main() + + def done(self, widget, data=None): + #self.destroy() + print "FINISHED!" + + def destroy(self, widget=None, data=None): + gtk.main_quit() + + + +# Test +if __name__ == '__main__': + # Three workers + worker1 = WorkerExample() + process1 = Process(worker1.go, ()) + observer1 = process1.getObserver() + + worker2 = WorkerExample() + process2 = Process(worker2.go, ()) + observer2 = process2.getObserver() + + worker3 = WorkerExample() + process3 = Process(worker3.go, ()) + observer3 = process3.getObserver() + + combo = CombinedObserver([observer1, observer2, observer3]) + + window0 = ProgressBarWindow(combo, "Workers' progress") + window1 = ProgressBarWindow(observer1, "Workers One's progress") + window2 = ProgressBarWindow(observer2, "Workers Two's progress") + window3 = ProgressBarWindow(observer3, "Workers Three's progress") + + process1.start() + #time.sleep(0.1) + process2.start() + #time.sleep(0.1) + process3.start() + gtk.main() + process1.join() + process2.join() + process3.join() + print "finished" + sys.exit() + + # Two workers + worker1 = WorkerExample() + process1 = Process(worker1.go, ()) + observer1 = process1.getObserver() + + worker2 = WorkerExample() + process2 = Process(worker2.go, ()) + observer2 = process2.getObserver() + + combo = CombinedObserver([observer1, observer2]) + + window0 = ProgressBarWindow(combo, "Workers' progress") + window1 = ProgressBarWindow(observer1, "Workers One's progress") + window2 = ProgressBarWindow(observer2, "Workers Two's progress") + + process1.start() + time.sleep(0.1) + process2.start() + gtk.main() + process1.join() + process2.join() + print "finished" + sys.exit() + + # One worker + print "creating worker1" + worker1 = WorkerExample() # The task to do, carefully constructed to give feedback via a queue + print "creating process1" + process1 = Process(worker1.go, ()) # Responsible for looking after the worker: supplying the queue, setting up the observer, starting process + observer1 = process1.getObserver() # An observer of the worker's progress that listens to the queue + window1 = ProgressBarWindow(observer1, "Workers One's progress") # A display of the progress, connected to the observer + process1.start() + gtk.main() + process1.join() + print "result = ", worker1.result + print "finished" + sys.exit() From 3613c79915e82ab983532b28a772982739df411d Mon Sep 17 00:00:00 2001 From: Jan Lerking Date: Sun, 4 Mar 2018 09:45:35 +0100 Subject: [PATCH 36/37] Added mp_test_4.py --- Multiprocessing test/mp_test_3.py | 27 +-- Multiprocessing test/mp_test_4.py | 300 ++++++++++++++++++++++++++++++ 2 files changed, 309 insertions(+), 18 deletions(-) create mode 100644 Multiprocessing test/mp_test_4.py diff --git a/Multiprocessing test/mp_test_3.py b/Multiprocessing test/mp_test_3.py index d9a6d62..f1f9787 100644 --- a/Multiprocessing test/mp_test_3.py +++ b/Multiprocessing test/mp_test_3.py @@ -8,10 +8,10 @@ import gtk import multiprocessing import threading import time - +import random + gtk.gdk.threads_init() - # Any time you connect to any of these signals, they should be # protected with a threading.Lock or similar. class ObserverBase(gobject.GObject): @@ -69,8 +69,7 @@ class Observer(ObserverBase): raise RuntimeError("Queue had an unknown data type ('"+dataType+"') with value '"+str(data)+"'") gobject.type_register(Observer) - - + class Process(gobject.GObject): __gsignals__ = { 'starting': (gobject.SIGNAL_RUN_LAST, @@ -93,7 +92,6 @@ class Process(gobject.GObject): # Create observer self.observer = Observer(self.queue) - def getObserver(self): return self.observer @@ -118,12 +116,8 @@ class Process(gobject.GObject): # Clean up queue #self.queue.close() - gobject.type_register(Process) - - - - + class CombinedObserver(ObserverBase): def __init__(self, observers, weights=None): ObserverBase.__init__(self) @@ -170,11 +164,8 @@ class CombinedObserver(ObserverBase): self.emit("errored") self.lock.release() - - gobject.type_register(CombinedObserver) - # A ProgressWorker is responsible for (a) getting the job done and (b) # reporting progress. Progress is reported via the queue. Fractions # (from 0 to 1) indicate the proportion of the job that is finished. @@ -190,7 +181,6 @@ class ProgressWorker(): def go(self): raise RuntimeError("Worker.go() has not been overridden") - class WorkerExample(ProgressWorker): def __init__(self): #Worker.__init__(self) @@ -206,7 +196,8 @@ class WorkerExample(ProgressWorker): for i in range(10): self.queue.put("working on step "+str(i)+"...") proportion = (float(i)+1)/10 - time.sleep(0.75) + slp = random.uniform(0.50, 3.50) + time.sleep(slp) self.queue.put(proportion) self.queue.put("done.") self.queue.put(1.0) @@ -316,9 +307,9 @@ if __name__ == '__main__': #time.sleep(0.1) process3.start() gtk.main() - process1.join() - process2.join() - process3.join() + #process1.join() + #process2.join() + #process3.join() print "finished" sys.exit() diff --git a/Multiprocessing test/mp_test_4.py b/Multiprocessing test/mp_test_4.py new file mode 100644 index 0000000..956ec83 --- /dev/null +++ b/Multiprocessing test/mp_test_4.py @@ -0,0 +1,300 @@ +#!/usr/bin/env python + +import sys #exit +import gobject +import pygtk +pygtk.require('2.0') +import gtk +import multiprocessing +import threading +import time +import random + +gtk.gdk.threads_init() + +# Any time you connect to any of these signals, they should be +# protected with a threading.Lock or similar. +class ObserverBase(gobject.GObject): + __gsignals__ = { + 'message_event' : (gobject.SIGNAL_RUN_LAST, + gobject.TYPE_NONE, + (gobject.TYPE_STRING,)), + 'updated' : (gobject.SIGNAL_RUN_LAST, + gobject.TYPE_NONE, + (gobject.TYPE_FLOAT,)), + 'finished': (gobject.SIGNAL_RUN_LAST, + gobject.TYPE_NONE, + ()), + 'errored': (gobject.SIGNAL_RUN_LAST, + gobject.TYPE_NONE, + ()) + } + + def __init__(self): + gobject.GObject.__init__(self) + +gobject.type_register(ObserverBase) + +class Observer(ObserverBase): + def __init__(self, queue): + ObserverBase.__init__(self) + self.queue = queue + + def start(self): + # Create new thread to do the listening + self.thread = threading.Thread(target=self.listen, args=()) + self.thread.start() + + def join(self): + self.thread.join() + + def listen(self): + while True: + # Listen for results on the queue and process them accordingly + print "Observer is waiting for data" + data = self.queue.get() + print "Observer received: ",data + dataType = type(data).__name__ + if dataType=="bool" and data==True: + self.emit('finished') + return + if dataType=="bool" and data==False: + self.emit('errored') + return + elif dataType=="str": + self.emit('message_event', data) + elif dataType=="float": + self.emit('updated', data) + else: + raise RuntimeError("Queue had an unknown data type ('"+dataType+"') with value '"+str(data)+"'") + +gobject.type_register(Observer) + +class Process(gobject.GObject): + __gsignals__ = { + 'starting': (gobject.SIGNAL_RUN_LAST, + gobject.TYPE_NONE, + ()), + 'joined': (gobject.SIGNAL_RUN_LAST, + gobject.TYPE_NONE, + ()) + } + + def __init__(self, function, arguments): + gobject.GObject.__init__(self) + # Create queue + self.queue = multiprocessing.Queue() + + # Create process + if arguments==(): arguments=(self.queue,) + self.process = multiprocessing.Process(target=function, args=arguments) + + # Create observer + self.observer = Observer(self.queue) + + def getObserver(self): + return self.observer + + def start(self): + # Starting observer + self.emit("starting") + print "starting observer" + self.observer.start() + # Start process + print "starting process" + self.process.start() + + def join(self): + # Clean up after process + print "cleaning up after process" + self.process.join() + self.emit("joined") + # Clean up after observer + print "cleaning up after observer" + self.queue.put(False) # If the false is ever read, it's an error + self.observer.join() + # Clean up queue + #self.queue.close() + +gobject.type_register(Process) + +class CombinedObserver(ObserverBase): + def __init__(self, observers, weights=None): + ObserverBase.__init__(self) + self.observers = observers + numObservers = len(observers) + if weights==None: + self.weights = [1 for i in range(numObservers)] + else: + self.weights = weights + + sum_weights = sum(self.weights) + self.weights = [float(w)/sum_weights for w in self.weights] + print "weights = ",self.weights + + index = 0 + for observer in self.observers: + #observer.connect("message_event", onMessageReceived, index) + observer.connect("updated", self.onUpdate, index) + observer.connect("finished", self.onFinished, index) + observer.connect("errored", self.onErrored, index) + index = index+1 + + self.fractions = [0 for i in range(numObservers)] + self.finished = [False for i in range(numObservers)] + self.errored = [False for i in range(numObservers)] + self.lock = threading.RLock() + + def onUpdate(self, widget, fraction, index): + self.lock.acquire() + self.fractions[index] = fraction * self.weights[index] + self.emit("updated", sum(self.fractions)) + self.lock.release() + + def onFinished(self, widget, index): + self.lock.acquire() + self.finished[index] = True + if all(self.finished): + self.emit("finished") + self.lock.release() + + def onErrored(self, widget, index): + self.lock.acquire() + self.errored[index] = True + self.emit("errored") + self.lock.release() + +gobject.type_register(CombinedObserver) + +# A ProgressWorker is responsible for (a) getting the job done and (b) +# reporting progress. Progress is reported via the queue. Fractions +# (from 0 to 1) indicate the proportion of the job that is finished. +# Strings may indicate what has been done or what is about to be done. +# The boolean 'True' is placed on the queue when the job is finished. +class ProgressWorker(): + def getResult(self): + return None + + def setQueue(self, queue): + self.queue = queue + + def go(self): + raise RuntimeError("Worker.go() has not been overridden") + +class WorkerExample(ProgressWorker): + def __init__(self): + #Worker.__init__(self) + manager = multiprocessing.Manager() + self.result = manager.list() + + def getResult(self): + return self.result + + def go(self, queue): + self.setQueue(queue) + print "The worker has started doing some work (counting from 0 to 9)" + for i in range(10): + self.queue.put("working on step "+str(i)+"...") + proportion = (float(i)+1)/10 + slp = random.uniform(0.50, 3.50) + time.sleep(slp) + self.queue.put(proportion) + self.queue.put("done.") + self.queue.put(1.0) + self.queue.put(True) + print "The worker has finished." + self.result.append("The the work has been finished and the result isn't 42.") + +class ProgressBar(gobject.GObject): + __gsignals__ = { + 'updated' : (gobject.SIGNAL_RUN_LAST, + gobject.TYPE_NONE, + (gobject.TYPE_FLOAT,)), + 'finished': (gobject.SIGNAL_RUN_LAST, + gobject.TYPE_NONE, + ()) + } + + def __init__(self, observer): + gobject.GObject.__init__(self) + self.observer = observer + self.observer.connect("updated",self.displayUpdate) + self.observer.connect("message_event",self.displayMessage) + self.observer.connect("finished",self.finish) + self.progress = gtk.ProgressBar() + self.progress.connect("destroy", self.destroy) + self.process=None + self.lock = threading.RLock() + + def displayUpdate(self, obj, fraction): + self.lock.acquire() + self.progress.set_fraction(fraction) + self.emit("updated", fraction) + self.lock.release() + + def displayMessage(self, obj, text): + self.lock.acquire() + self.progress.set_text(text) + self.lock.release() + + def finish(self, obj, data=None): + self.emit("finished") + + def destroy(self, widget, data=None): + print "Destroying" + if self.process==None: + return + self.queue.put("finished") + self.process.join() + self.thread.join() + +gobject.type_register(ProgressBar) + +class ProgressBarWindow: + def __init__(self, observer, title): + self.progressBar = ProgressBar(observer) + self.progressBar.connect("finished", self.done) + window = gtk.Window(gtk.WINDOW_TOPLEVEL) + window.set_border_width(10) + window.set_title(title) + window.add(self.progressBar.progress) + window.show_all() + window.connect("destroy", self.destroy) + + def main(self): + gtk.main() + + def done(self, widget, data=None): + #self.destroy() + print "FINISHED!" + + def destroy(self, widget=None, data=None): + gtk.main_quit() + +# Test +if __name__ == '__main__': + # n workers + worker = [] + process = [] + observer =[] + window = [] + + for i in range(10): + worker.append(WorkerExample()) + process.append(Process(worker[i].go, ())) + observer.append(Process[i].getObserver()) + + combo = CombinedObserver(observer) + + window0 = ProgressBarWindow(combo, "Workers' progress") + for i in range(10): + window.append(ProgressBarWindow(observer[i], "Workers {}'s progress".format(str(i)))) + + for i in range(10): + process[i].start() + + gtk.main() + + print "finished" + sys.exit() + \ No newline at end of file From 30c2aa3605717ec6d8dfb52857a70a007858e7e5 Mon Sep 17 00:00:00 2001 From: Lerking <33354709+Lerking@users.noreply.github.com> Date: Thu, 8 Mar 2018 12:54:06 +0100 Subject: [PATCH 37/37] Added concurrency-futures.py --- concurrency-futures.py | 50 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 concurrency-futures.py diff --git a/concurrency-futures.py b/concurrency-futures.py new file mode 100644 index 0000000..d253ab7 --- /dev/null +++ b/concurrency-futures.py @@ -0,0 +1,50 @@ +import concurrent.futures +import time +import multiprocessing + +num_cores = multiprocessing.cpu_count() + +def is_prime(num): + if num <= 1: + return False + elif num <= 3: + return True + elif num%2 == 0 or num%3 == 0: + return False + i = 5 + while i*i <= num: + if num%i == 0 or num%(i+2) == 0: + return False + i += 6 + return True + +def find_sum(num): + sum_of_primes = 0 + + ix = 2 + + while ix <= num: + if is_prime(ix): + sum_of_primes += ix + ix += 1 + print("{} : Sum = {}".format(num, sum_of_primes)) + return sum_of_primes + +def sum_primes_thread(nums): + with concurrent.futures.ThreadPoolExecutor(max_workers = num_cores) as executor: + executor.map(find_sum, nums) + +def sum_primes_process(nums): + with concurrent.futures.ProcessPoolExecutor(max_workers = num_cores) as executor: + executor.map(find_sum, nums) + +if __name__ == '__main__': + nums = [1000000, 900000, 800000, 700000, 600000, 500000, 400000, 300000, 200000, 100000] + print("Using multithreading (num_threads = num_cores):") + start = time.time() + sum_primes_thread(nums) + print("Time taken = {0:.5f}".format(time.time() - start)) + print("Using multiprocessing (num_processes = num_cores):") + start = time.time() + sum_primes_process(nums) + print("Time taken = {0:.5f}".format(time.time() - start)) \ No newline at end of file