Added multi threading

multiprocessing_test.py

@@ -11,12 +11,41 @@ import multiprocessing
 import time
 import io
 import cpuinfo
+import queue
+import threading
+import time
 
 cpu_info = cpuinfo.get_cpu_info()
 num_cores = multiprocessing.cpu_count()
 print(num_cores)
 
 filelist = []
+exitFlag = 0
+queueLock = threading.Lock()
+workQueue = queue.Queue(0)
+threads = []
+class myThread (threading.Thread):
+    def __init__(self, threadID, name, q):
+        threading.Thread.__init__(self)
+        self.threadID = threadID
+        self.name = name
+        self.q = q
+    def run(self):
+        print ("Starting " + self.name)
+        process_data(self.name, self.q)
+        print ("Exiting " + self.name)
+
+def process_data(threadName, q):
+    while not exitFlag:
+        queueLock.acquire()
+        if not workQueue.empty():
+            data = q.get()
+            process_file(data)
+            queueLock.release()
+            print ("%s processing %s" % (threadName, data))
+        else:
+            queueLock.release()
+            time.sleep(1)
 
 def sourcedir_filecnt(sourcedir):
     ### Return the number of files, ending with '.h', in sourcedir - including subdirectories ###
@@ -32,8 +61,11 @@ def sourcedir_filecnt(sourcedir):
 def process_files(sourcedir, destdir):
     global filelist
     for f in filelist:
+        process_file(f)
+
+def process_file(data):
     outfile = ''
-        inputfile = f
+    inputfile = data
     encodings = ['utf-8', 'latin-1', 'windows-1250', 'windows-1252', 'ascii',
                 'big5', 'big5hkscs', 'cp037', 'cp273', 'cp424', 'cp437', 'cp500',
                 'cp720', 'cp737', 'cp775', 'cp850', 'cp852', 'cp855', 'cp856',
@@ -52,7 +84,7 @@ def process_files(sourcedir, destdir):
                 'utf-32-le', 'utf-16', 'utf-16-be', 'utf-16-le', 'utf-7', 'utf-8-sig']
     for e in encodings:
         try:
-                fh = io.open(f, 'r', encoding=e)
+            fh = io.open(data, 'r', encoding=e)
             fh.readlines()
             fh.seek(0)
         except UnicodeDecodeError:
@@ -60,7 +92,7 @@ def process_files(sourcedir, destdir):
         else:
             print('opening the file with encoding:  %s ' % e)
             break 
-        print(os.path.basename(f))
+    print(os.path.basename(data))
     for lines in fh:
         outfile = outfile+lines
     fh.close()
@@ -84,10 +116,50 @@ def single_thread():
     process_files(sourcedir, destdir)
     print('Single thread process time: '+str(time.time()-t1))
 
+def multi_thread():
+    global filelist
+    global exitFlag
+
+    t1 = time.time()
+    cnt = sourcedir_filecnt(sourcedir)
+    threadList = ["thread 1", "thread 2", "thread 3", "thread 4", "thread 5"]
+    nameList = filelist
+    threadID = 0
+
+    # Create new threads
+    for tName in threadList:
+        thread = myThread(threadID, tName, workQueue)
+        thread.start()
+        threads.append(thread)
+        threadID += 1
+
+    # Fill the queue
+    queueLock.acquire()
+    for word in nameList:
+        workQueue.put(word)
+    queueLock.release()
+
+    # Wait for queue to empty
+    while not workQueue.empty():
+        pass
+
+    # Notify threads it's time to exit
+    exitFlag = 1
+
+    # Wait for all threads to complete
+    for t in threads:
+        t.join()
+    print ("Exiting Main Thread")
+    print('Files in: '+str(cnt))
+    print('Multi thread process time: '+str(time.time()-t1))
+
 #sourcedir = 'C:/Users/dksojlg/Documents/gtk+-3.22.26'
 sourcedir = '/usr/include'
 #destdir = 'C:/Users/dksojlg/Documents/include'
 destdir = '/data_2/include'
 
 print(cpu_info)
-single_thread() #2543 files - Single thread process time: 32.53886866569519
+single_thread() #2543 files - Single thread process time: 1.3732633590698242
+
+multi_thread() 
+