diff --git a/multiprocessing_test.py b/multiprocessing_test.py index 9b73d04..931843d 100755 --- a/multiprocessing_test.py +++ b/multiprocessing_test.py @@ -9,16 +9,19 @@ import os import sys -import multiprocessing +import multiprocessing as mp import time import io import cpuinfo import queue import threading import time +import random +import string +import shutil cpu_info = cpuinfo.get_cpu_info() -num_cores = multiprocessing.cpu_count() +num_cores = mp.cpu_count() print(num_cores) filelist = [] @@ -92,15 +95,15 @@ def process_file(data): except UnicodeDecodeError: print('got unicode error with %s , trying different encoding' % e) else: - print('opening the file with encoding: %s ' % e) + #print('opening the file with encoding: %s ' % e) break - print(os.path.basename(data)) + #print(os.path.basename(data)) for lines in fh: outfile = outfile+lines fh.close() outputfile = os.path.splitext(inputfile)[0]+'.inc' outputfile = str(outputfile).replace(sourcedir, destdir) - print(outputfile) + #print(outputfile) if not os.path.exists(os.path.dirname(outputfile)): try: os.makedirs(os.path.dirname(outputfile)) @@ -118,7 +121,7 @@ def single_thread(): process_files(sourcedir, destdir) print('Single thread process time: '+str(time.time()-t1)) -def syncronized_thread(): +def multi_thread(): global filelist global exitFlag @@ -155,13 +158,53 @@ def syncronized_thread(): print('Files in: '+str(cnt)) print('Multi thread process time: '+str(time.time()-t1)) +def async_process(num): + t1 = time.time() + cnt = sourcedir_filecnt(sourcedir) + + pool = mp.Pool(processes=num) + #[pool.apply_async(process_file, args=(f,)) for f in filelist] + pool.map(process_file, filelist) + #output = [p.get() for p in results] + #print(output) + + # Setup a list of processes that we want to run + #processes = [mp.Process(target=rand_string, args=(5, output)) for x in range(8)] + #processes = [mp.Process(target=process_files, args=(sourcedir, destdir)) for x in range(num)] + + # Run processes + #for p in processes: + #p.start() + + # Exit the completed processes + #for p in processes: + #p.join() + + # Get process results from the output queue + #results = [output.get() for p in processes] + + #print(results) + print('Files in: '+str(cnt)) + print('Multi ('+str(num)+') processes time: '+str(time.time()-t1)) + +def cleanup(dest): + shutil.rmtree(dest) + #sourcedir = 'C:/Users/dksojlg/Documents/gtk+-3.22.26' sourcedir = '/usr/include' #destdir = 'C:/Users/dksojlg/Documents/include' destdir = '/data_2/include' -print(cpu_info) -single_thread() #2543 files - Single thread process time: 1.3732633590698242 sec. - -syncronized_thread() #2543 files - Syncronized 5 threads process time: 80.25179100036621 sec. - +#print(cpu_info) +#single_thread() #2543 files - Single thread process time: 1.3732633590698242 sec. +#os.rmdir(destdir) +#multi_thread() #2543 files - Syncronized 5 threads process time: 80.25179100036621 sec. +#os.rmdir(destdir) +async_process(16) #2543 files - 2 processes, process time: 1.2379400730133057 sec. +#cleanup(destdir) +#async_process(4) #2543 files - 2 processes, process time: 2.6622860431671143 sec. +#cleanup(destdir) +#async_process(6) #2543 files - 2 processes, process time: 2.6622860431671143 sec. +#cleanup(destdir) +#async_process(8) #2543 files - 2 processes, process time: 2.6622860431671143 sec. +#cleanup(destdir)