diff --git a/multiprocessing_test.py b/multiprocessing_test.py old mode 100644 new mode 100755 index 452aa5d..bc5a4b8 --- a/multiprocessing_test.py +++ b/multiprocessing_test.py @@ -1,165 +1,167 @@ -# You are free to use and/or change this code for -# your own needs. - -# Original code (c)2018 Jan Lerking -# Program to test various scenarios of -# single thread, multi thread, pool and process - -import os -import sys -import multiprocessing -import time -import io -import cpuinfo -import queue -import threading -import time - -cpu_info = cpuinfo.get_cpu_info() -num_cores = multiprocessing.cpu_count() -print(num_cores) - -filelist = [] -exitFlag = 0 -queueLock = threading.Lock() -workQueue = queue.Queue(0) -threads = [] -class myThread (threading.Thread): - def __init__(self, threadID, name, q): - threading.Thread.__init__(self) - self.threadID = threadID - self.name = name - self.q = q - def run(self): - print ("Starting " + self.name) - process_data(self.name, self.q) - print ("Exiting " + self.name) - -def process_data(threadName, q): - while not exitFlag: - queueLock.acquire() - if not workQueue.empty(): - data = q.get() - process_file(data) - queueLock.release() - print ("%s processing %s" % (threadName, data)) - else: - queueLock.release() - time.sleep(1) - -def sourcedir_filecnt(sourcedir): - ### Return the number of files, ending with '.h', in sourcedir - including subdirectories ### - cnt = 0 - global filelist - for folderName, subfolders, files in os.walk(sourcedir): - for file in files: - if file.lower().endswith('.h'): - cnt += 1 - filelist += [folderName+'/'+file] - return cnt - -def process_files(sourcedir, destdir): - global filelist - for f in filelist: - process_file(f) - -def process_file(data): - outfile = '' - inputfile = data - encodings = ['utf-8', 'latin-1', 'windows-1250', 'windows-1252', 'ascii', - 'big5', 'big5hkscs', 'cp037', 'cp273', 'cp424', 'cp437', 'cp500', - 'cp720', 'cp737', 'cp775', 'cp850', 'cp852', 'cp855', 'cp856', - 'cp857', 'cp858', 'cp860', 'cp861', 'cp862', 'cp863', 'cp864', 'cp865', - 'cp866', 'cp869', 'cp874', 'cp875', 'cp932', 'cp949', 'cp950', 'cp1006', - 'cp1026', 'cp1125', 'cp1140', 'cp1250', 'cp1251', 'cp1252', 'cp1253', 'cp1254', - 'cp1255', 'cp1256', 'cp1257', 'cp1258', 'cp65001', 'euc-jp', 'euc-jis-2004', - 'euc-jisx0213', 'euc-kr', 'gb2312', 'gbk', 'gb18030', 'hz', 'iso2022-jp', - 'iso2022-jp-1', 'iso2022-jp-2', 'iso2022-jp-2004', 'iso2022-jp-3', - 'iso2022-jp-ext', 'iso2022-kr', 'iso8859-2', 'iso8859-3', 'iso8859-4', - 'iso8859-5', 'iso8859-6', 'iso8859-7', 'iso8859-8', 'iso8859-9', 'iso8859-10', - 'iso8859-11', 'iso8859-13', 'iso8859-14', 'iso8859-15', 'iso8859-16', 'johab', - 'koi8-r', 'koi8-t', 'koi8-u', 'kz1048', 'mac-cyrillic', 'mac-greek', - 'mac-iceland', 'mac-latin2', 'mac-roman', 'mac-turkish', 'ptcp154', - 'shift-jis', 'shift-jis-2004', 'shift-jisx0213', 'utf-32', 'utf-32-be', - 'utf-32-le', 'utf-16', 'utf-16-be', 'utf-16-le', 'utf-7', 'utf-8-sig'] - for e in encodings: - try: - fh = io.open(data, 'r', encoding=e) - fh.readlines() - fh.seek(0) - except UnicodeDecodeError: - print('got unicode error with %s , trying different encoding' % e) - else: - print('opening the file with encoding: %s ' % e) - break - print(os.path.basename(data)) - for lines in fh: - outfile = outfile+lines - fh.close() - outputfile = os.path.splitext(inputfile)[0]+'.inc' - outputfile = str(outputfile).replace(sourcedir, destdir) - print(outputfile) - if not os.path.exists(os.path.dirname(outputfile)): - try: - os.makedirs(os.path.dirname(outputfile)) - except OSError as exc: # Guard against race condition - if exc.errno != errno.EEXIST: - raise - newfile = open(outputfile, "w") - newfile.write(outfile) - newfile.close() - -def single_thread(): - t1 = time.time() - cnt = sourcedir_filecnt(sourcedir) - print('Files in: '+str(cnt)) - process_files(sourcedir, destdir) - print('Single thread process time: '+str(time.time()-t1)) - -def multi_thread(): - global filelist - global exitFlag - - t1 = time.time() - cnt = sourcedir_filecnt(sourcedir) - threadList = ["thread 1", "thread 2", "thread 3", "thread 4", "thread 5"] - nameList = filelist - threadID = 0 - - # Create new threads - for tName in threadList: - thread = myThread(threadID, tName, workQueue) - thread.start() - threads.append(thread) - threadID += 1 - - # Fill the queue - queueLock.acquire() - for word in nameList: - workQueue.put(word) - queueLock.release() - - # Wait for queue to empty - while not workQueue.empty(): - pass - - # Notify threads it's time to exit - exitFlag = 1 - - # Wait for all threads to complete - for t in threads: - t.join() - print ("Exiting Main Thread") - print('Files in: '+str(cnt)) - print('Multi thread process time: '+str(time.time()-t1)) - -#sourcedir = 'C:/Users/dksojlg/Documents/gtk+-3.22.26' -sourcedir = '/usr/include' -#destdir = 'C:/Users/dksojlg/Documents/include' -destdir = '/data_2/include' - -print(cpu_info) -single_thread() #2543 files - Single thread process time: 1.3732633590698242 - -multi_thread() - +#!/usr/bin/env python3.5 + +# You are free to use and/or change this code for +# your own needs. + +# Original code (c)2018 Jan Lerking +# Program to test various scenarios of +# single thread, multi thread, pool and process + +import os +import sys +import multiprocessing +import time +import io +import cpuinfo +import queue +import threading +import time + +cpu_info = cpuinfo.get_cpu_info() +num_cores = multiprocessing.cpu_count() +print(num_cores) + +filelist = [] +exitFlag = 0 +queueLock = threading.Lock() +workQueue = queue.Queue(0) +threads = [] +class myThread (threading.Thread): + def __init__(self, threadID, name, q): + threading.Thread.__init__(self) + self.threadID = threadID + self.name = name + self.q = q + def run(self): + print ("Starting " + self.name) + process_data(self.name, self.q) + print ("Exiting " + self.name) + +def process_data(threadName, q): + while not exitFlag: + queueLock.acquire() + if not workQueue.empty(): + data = q.get() + process_file(data) + queueLock.release() + print ("%s processing %s" % (threadName, data)) + else: + queueLock.release() + time.sleep(1) + +def sourcedir_filecnt(sourcedir): + ### Return the number of files, ending with '.h', in sourcedir - including subdirectories ### + cnt = 0 + global filelist + for folderName, subfolders, files in os.walk(sourcedir): + for file in files: + if file.lower().endswith('.h'): + cnt += 1 + filelist += [folderName+'/'+file] + return cnt + +def process_files(sourcedir, destdir): + global filelist + for f in filelist: + process_file(f) + +def process_file(data): + outfile = '' + inputfile = data + encodings = ['utf-8', 'latin-1', 'windows-1250', 'windows-1252', 'ascii', + 'big5', 'big5hkscs', 'cp037', 'cp273', 'cp424', 'cp437', 'cp500', + 'cp720', 'cp737', 'cp775', 'cp850', 'cp852', 'cp855', 'cp856', + 'cp857', 'cp858', 'cp860', 'cp861', 'cp862', 'cp863', 'cp864', 'cp865', + 'cp866', 'cp869', 'cp874', 'cp875', 'cp932', 'cp949', 'cp950', 'cp1006', + 'cp1026', 'cp1125', 'cp1140', 'cp1250', 'cp1251', 'cp1252', 'cp1253', 'cp1254', + 'cp1255', 'cp1256', 'cp1257', 'cp1258', 'cp65001', 'euc-jp', 'euc-jis-2004', + 'euc-jisx0213', 'euc-kr', 'gb2312', 'gbk', 'gb18030', 'hz', 'iso2022-jp', + 'iso2022-jp-1', 'iso2022-jp-2', 'iso2022-jp-2004', 'iso2022-jp-3', + 'iso2022-jp-ext', 'iso2022-kr', 'iso8859-2', 'iso8859-3', 'iso8859-4', + 'iso8859-5', 'iso8859-6', 'iso8859-7', 'iso8859-8', 'iso8859-9', 'iso8859-10', + 'iso8859-11', 'iso8859-13', 'iso8859-14', 'iso8859-15', 'iso8859-16', 'johab', + 'koi8-r', 'koi8-t', 'koi8-u', 'kz1048', 'mac-cyrillic', 'mac-greek', + 'mac-iceland', 'mac-latin2', 'mac-roman', 'mac-turkish', 'ptcp154', + 'shift-jis', 'shift-jis-2004', 'shift-jisx0213', 'utf-32', 'utf-32-be', + 'utf-32-le', 'utf-16', 'utf-16-be', 'utf-16-le', 'utf-7', 'utf-8-sig'] + for e in encodings: + try: + fh = io.open(data, 'r', encoding=e) + fh.readlines() + fh.seek(0) + except UnicodeDecodeError: + print('got unicode error with %s , trying different encoding' % e) + else: + print('opening the file with encoding: %s ' % e) + break + print(os.path.basename(data)) + for lines in fh: + outfile = outfile+lines + fh.close() + outputfile = os.path.splitext(inputfile)[0]+'.inc' + outputfile = str(outputfile).replace(sourcedir, destdir) + print(outputfile) + if not os.path.exists(os.path.dirname(outputfile)): + try: + os.makedirs(os.path.dirname(outputfile)) + except OSError as exc: # Guard against race condition + if exc.errno != errno.EEXIST: + raise + newfile = open(outputfile, "w") + newfile.write(outfile) + newfile.close() + +def single_thread(): + t1 = time.time() + cnt = sourcedir_filecnt(sourcedir) + print('Files in: '+str(cnt)) + process_files(sourcedir, destdir) + print('Single thread process time: '+str(time.time()-t1)) + +def multi_thread(): + global filelist + global exitFlag + + t1 = time.time() + cnt = sourcedir_filecnt(sourcedir) + threadList = ["thread 1", "thread 2", "thread 3", "thread 4", "thread 5"] + nameList = filelist + threadID = 0 + + # Create new threads + for tName in threadList: + thread = myThread(threadID, tName, workQueue) + thread.start() + threads.append(thread) + threadID += 1 + + # Fill the queue + queueLock.acquire() + for word in nameList: + workQueue.put(word) + queueLock.release() + + # Wait for queue to empty + while not workQueue.empty(): + pass + + # Notify threads it's time to exit + exitFlag = 1 + + # Wait for all threads to complete + for t in threads: + t.join() + print ("Exiting Main Thread") + print('Files in: '+str(cnt)) + print('Multi thread process time: '+str(time.time()-t1)) + +#sourcedir = 'C:/Users/dksojlg/Documents/gtk+-3.22.26' +sourcedir = '/usr/include' +#destdir = 'C:/Users/dksojlg/Documents/include' +destdir = '/data_2/include' + +print(cpu_info) +single_thread() #2543 files - Single thread process time: 1.3732633590698242 + +multi_thread() +