Updated h2inc to use multiprocessing

async_multiprocessing_test.py

@@ -0,0 +1,34 @@
+import multiprocessing as mp
+import random
+import string
+
+random.seed(123)
+
+# Define an output queue
+output = mp.Queue()
+
+# define a example function
+def rand_string(length, output):
+    """ Generates a random string of numbers, lower- and uppercase chars. """
+    rand_str = ''.join(random.choice(
+                        string.ascii_lowercase
+                        + string.ascii_uppercase
+                        + string.digits)
+                   for i in range(length))
+    output.put(rand_str)
+
+# Setup a list of processes that we want to run
+processes = [mp.Process(target=rand_string, args=(5, output)) for x in range(4)]
+
+# Run processes
+for p in processes:
+    p.start()
+
+# Exit the completed processes
+for p in processes:
+    p.join()
+
+# Get process results from the output queue
+results = [output.get() for p in processes]
+
+print(results)

h2inc

@@ -0,0 +1 @@
+/data_2/Projects/h2inc/h2inc.py

h2inc.py

@@ -1,87 +1,112 @@
-# You are free to use and/or change this code for
+#!/usr/bin/env python3.5
-# your own needs.
+
-
+# You are free to use and/or change this code for
-# Original code (c)2018 Jan Lerking
+# your own needs.
-# Program to convert C-header (*.h) files to nasm include files (*.inc),
+
-# for direct usage in assembly programming using nasm/yasm.
+# Original code (c)2018 Jan Lerking
-
+# Program to convert C-header (*.h) files to nasm include files (*.inc),
-import os
+# for direct usage in assembly programming using nasm/yasm.
-import sys
+
-from h2inc_parser import parseline, parseparsed
+import os
-import h2inc_gui
+import sys
-
+import multiprocessing as mp
-tupline = []
+import io
-preproc = ()
+from h2inc_parser import parseline, parseparsed
-filelist = []
+import h2inc_gui
-folderlist = []
+
-cnt = 0
+tupline = []
-
+preproc = ()
-def sourcedir_filecnt(sourcedir):
+filelist = []
-    ### Return the number of files, ending with '.h', in sourcedir - including subdirectories ###
+folderlist = []
-    cnt = 0
+cnt = 0
-    global filelist
+sourcedir = ''
-    for folderName, subfolders, files in os.walk(sourcedir):
+destdir = ''
-        for file in files:
+num_cores = mp.cpu_count()
-            if file.lower().endswith('.h'):
+
-                cnt += 1
+def sourcedir_filecnt(sourcedir):
-                filelist += [folderName+'/'+file]
+    ### Return the number of files, ending with '.h', in sourcedir - including subdirectories ###
-                print(folderName+'/'+file)
+    cnt = 0
-    #print(filelist)
+    global filelist
-    return cnt
+    for folderName, subfolders, files in os.walk(sourcedir):
-                
+        for file in files:
-def sourcedir_foldercnt(sourcedir):
+            if file.lower().endswith('.h'):
-    ### Return the number of folders, if it contains '*.h' files, in sourcedir - including subdirectories ###
+                cnt += 1
-    global cnt
+                filelist += [folderName+'/'+file]
-    global folderlist
+                print(folderName+'/'+file)
-    for folderName, subfolders, files in os.walk(sourcedir):
+    #print(filelist)
-        if subfolders:
+    return cnt
-            for subfolder in subfolders:
+                
-                sourcedir_foldercnt(subfolder)
+def sourcedir_foldercnt(sourcedir):
-        tempf = [file for file in files if file.lower().endswith('.h')]
+    ### Return the number of folders, if it contains '*.h' files, in sourcedir - including subdirectories ###
-        if tempf:
+    global cnt
-            cnt = cnt+1
+    global folderlist
-            #print(folderName)
+    for folderName, subfolders, files in os.walk(sourcedir):
-            folderlist += [folderName]
+        if subfolders:
-    #print(folderlist)
+            for subfolder in subfolders:
-    #print(len(folderlist))
+                sourcedir_foldercnt(subfolder)
-    return cnt
+        tempf = [file for file in files if file.lower().endswith('.h')]
-
+        if tempf:
-def process_files(sourcedir, destdir, filevar):
+            cnt = cnt+1
-    #global folderlist
+            #print(folderName)
-    global filelist
+            folderlist += [folderName]
-    global preproc
+    #print(folderlist)
-    outfile = ''
+    #print(len(folderlist))
-    cnt = 0
+    return cnt
-    for f in filelist:
+
-        current_cnt = 0
+def process_files(source, dest):
-        inputfile = f
+    global sourcedir
-        filehandle = open(f, 'r')
+    global destdir
-        current_cnt += 1
+    sourcedir = source
-        filevar = current_cnt
+    destdir = dest
-        print(os.path.basename(f))
+    pool = mp.Pool(processes=num_cores)
-        currentfile_update(os.path.basename(f))
+    pool.map(process_file, filelist)
-        for lines in filehandle:
+
-            print(lines)
+def process_file(data):
-            preproc = preproc+tuple([parseline(lines)])
+    outfile = ''
-        filehandle.close()
+    inputfile = data
-        current_cnt += 1
+    encodings = ['utf-8', 'latin-1', 'windows-1250', 'windows-1252', 'ascii',
-        filevar = current_cnt
+                'big5', 'big5hkscs', 'cp037', 'cp273', 'cp424', 'cp437', 'cp500',
-        for elements in preproc:
+                'cp720', 'cp737', 'cp775', 'cp850', 'cp852', 'cp855', 'cp856',
-            outfile = outfile+parseparsed(elements)
+                'cp857', 'cp858', 'cp860', 'cp861', 'cp862', 'cp863', 'cp864', 'cp865',
-        outputfile = os.path.splitext(inputfile)[0]+'.inc'
+                'cp866', 'cp869', 'cp874', 'cp875', 'cp932', 'cp949', 'cp950', 'cp1006',
-        outputfile = str(outputfile).replace(str(sourcedir.get()), str(destdir.get()))
+                'cp1026', 'cp1125', 'cp1140', 'cp1250', 'cp1251', 'cp1252', 'cp1253', 'cp1254',
-        current_cnt += 1
+                'cp1255', 'cp1256', 'cp1257', 'cp1258', 'cp65001', 'euc-jp', 'euc-jis-2004',
-        filevar = current_cnt
+                'euc-jisx0213', 'euc-kr', 'gb2312', 'gbk', 'gb18030', 'hz', 'iso2022-jp',
-        print(outputfile)
+                'iso2022-jp-1', 'iso2022-jp-2', 'iso2022-jp-2004', 'iso2022-jp-3',
-        if not os.path.exists(os.path.dirname(outputfile)):
+                'iso2022-jp-ext', 'iso2022-kr', 'iso8859-2', 'iso8859-3', 'iso8859-4',
-            try:  
+                'iso8859-5', 'iso8859-6', 'iso8859-7', 'iso8859-8', 'iso8859-9', 'iso8859-10',
-                os.makedirs(os.path.dirname(outputfile))
+                'iso8859-11', 'iso8859-13', 'iso8859-14', 'iso8859-15', 'iso8859-16', 'johab',
-            except OSError as exc: # Guard against race condition
+                'koi8-r', 'koi8-t', 'koi8-u', 'kz1048', 'mac-cyrillic', 'mac-greek',
-                if exc.errno != errno.EEXIST:
+                'mac-iceland', 'mac-latin2', 'mac-roman', 'mac-turkish', 'ptcp154',
-                    raise
+                'shift-jis', 'shift-jis-2004', 'shift-jisx0213', 'utf-32', 'utf-32-be',
-        newfile = open(outputfile, "w")
+                'utf-32-le', 'utf-16', 'utf-16-be', 'utf-16-le', 'utf-7', 'utf-8-sig']
-        newfile.write(outfile)
+    for e in encodings:
-        newfile.close()
+        try:
-        current_cnt += 1
+            fh = io.open(data, 'r', encoding=e)
-        filevar = current_cnt
+            fh.readlines()
-        cnt += 1
+            fh.seek(0)
+        except UnicodeDecodeError:
+            print('got unicode error with %s , trying different encoding' % e)
+        else:
+            #print('opening the file with encoding:  %s ' % e)
+            break 
+    #print(os.path.basename(data))
+    for lines in fh:
+        outfile = outfile+lines
+    fh.close()
+    outputfile = os.path.splitext(inputfile)[0]+'.inc'
+    outputfile = str(outputfile).replace(sourcedir, destdir)
+    #print(outputfile)
+    if not os.path.exists(os.path.dirname(outputfile)):
+        try:  
+            os.makedirs(os.path.dirname(outputfile))
+        except OSError as exc: # Guard against race condition
+            if exc.errno != errno.EEXIST:
+                raise
+    newfile = open(outputfile, "w")
+    newfile.write(outfile)
+    newfile.close()
+
+def async_process(num):
+    pool = mp.Pool(processes=num)
+    pool.map(process_file, filelist)

h2inc_gui.py

@@ -142,6 +142,7 @@ class h2incGUI:
     def translate(self, destdir, sourcedir, addinc):
         doinc = addinc.get()
         dest = destdir.get()
+        source = sourcedir.get()
         if doinc == 'yes':
             dest = dest+'/include'
             print(os.path.exists(os.path.dirname(dest)))
@@ -153,7 +154,7 @@ class h2incGUI:
                         raise
         destdir.set(dest)
         print ('Destination directory: ', destdir.get())
-        process_files(sourcedir, destdir, self.cfilevar)
+        process_files(source, dest)
         
     def cfileprogress_update(cnt):
         self.cfilevar = cnt

multiprocessing

@@ -0,0 +1 @@
+/data_2/Projects/h2inc/multiprocessing_test.py

pool_test.py

@@ -0,0 +1,20 @@
+from multiprocessing import Pool
+
+import time
+
+work = (["A", 5], ["B", 2], ["C", 1], ["D", 3])
+
+
+def work_log(work_data):
+    print(" Process %s waiting %s seconds" % (work_data[0], work_data[1]))
+    time.sleep(int(work_data[1]))
+    print(" Process %s Finished." % work_data[0])
+
+
+def pool_handler():
+    p = Pool(2)
+    p.map(work_log, work)
+
+
+if __name__ == '__main__':
+    pool_handler()