Added pool.map testing

multiprocessing_test.py

@@ -9,16 +9,19 @@
 
 import os
 import sys
-import multiprocessing
+import multiprocessing as mp
 import time
 import io
 import cpuinfo
 import queue
 import threading
 import time
+import random
+import string
+import shutil
 
 cpu_info = cpuinfo.get_cpu_info()
-num_cores = multiprocessing.cpu_count()
+num_cores = mp.cpu_count()
 print(num_cores)
 
 filelist = []
@@ -92,15 +95,15 @@ def process_file(data):
         except UnicodeDecodeError:
             print('got unicode error with %s , trying different encoding' % e)
         else:
-            print('opening the file with encoding:  %s ' % e)
+            #print('opening the file with encoding:  %s ' % e)
             break 
-    print(os.path.basename(data))
+    #print(os.path.basename(data))
     for lines in fh:
         outfile = outfile+lines
     fh.close()
     outputfile = os.path.splitext(inputfile)[0]+'.inc'
     outputfile = str(outputfile).replace(sourcedir, destdir)
-    print(outputfile)
+    #print(outputfile)
     if not os.path.exists(os.path.dirname(outputfile)):
         try:  
             os.makedirs(os.path.dirname(outputfile))
@@ -118,7 +121,7 @@ def single_thread():
     process_files(sourcedir, destdir)
     print('Single thread process time: '+str(time.time()-t1))
 
-def syncronized_thread():
+def multi_thread():
     global filelist
     global exitFlag
 
@@ -155,13 +158,53 @@ def syncronized_thread():
     print('Files in: '+str(cnt))
     print('Multi thread process time: '+str(time.time()-t1))
 
+def async_process(num):
+    t1 = time.time()
+    cnt = sourcedir_filecnt(sourcedir)
+    
+    pool = mp.Pool(processes=num)
+    #[pool.apply_async(process_file, args=(f,)) for f in filelist]
+    pool.map(process_file, filelist)
+    #output = [p.get() for p in results]
+    #print(output)
+
+    # Setup a list of processes that we want to run
+    #processes = [mp.Process(target=rand_string, args=(5, output)) for x in range(8)]
+    #processes = [mp.Process(target=process_files, args=(sourcedir, destdir)) for x in range(num)]
+
+    # Run processes
+    #for p in processes:
+        #p.start()
+
+    # Exit the completed processes
+    #for p in processes:
+        #p.join()
+
+    # Get process results from the output queue
+    #results = [output.get() for p in processes]
+
+    #print(results)
+    print('Files in: '+str(cnt))
+    print('Multi ('+str(num)+') processes time: '+str(time.time()-t1))
+
+def cleanup(dest):
+    shutil.rmtree(dest)
+
 #sourcedir = 'C:/Users/dksojlg/Documents/gtk+-3.22.26'
 sourcedir = '/usr/include'
 #destdir = 'C:/Users/dksojlg/Documents/include'
 destdir = '/data_2/include'
 
-print(cpu_info)
+#print(cpu_info)
-single_thread() #2543 files - Single thread process time: 1.3732633590698242 sec.
+#single_thread() #2543 files - Single thread process time: 1.3732633590698242 sec.
-
+#os.rmdir(destdir)
-syncronized_thread() #2543 files - Syncronized 5 threads process time: 80.25179100036621 sec.
+#multi_thread() #2543 files - Syncronized 5 threads process time: 80.25179100036621 sec.
-
+#os.rmdir(destdir)
+async_process(16) #2543 files - 2 processes, process time: 1.2379400730133057 sec.
+#cleanup(destdir)
+#async_process(4) #2543 files - 2 processes, process time: 2.6622860431671143 sec.
+#cleanup(destdir)
+#async_process(6) #2543 files - 2 processes, process time: 2.6622860431671143 sec.
+#cleanup(destdir)
+#async_process(8) #2543 files - 2 processes, process time: 2.6622860431671143 sec.
+#cleanup(destdir)