ee

parser/parser.py

@@ -2,20 +2,21 @@
 # Python parser for c header files.
 # Used for creating corresponding NASM include files.
 
-import keywords
 import os
 import sys
 import multiprocessing
 import threading
+import time
+import keywords
 
 keywords.init()
 num_cores = multiprocessing.cpu_count()
 work_queue = multiprocessing.JoinableQueue()
 signal_queue = multiprocessing.JoinableQueue()
 
-test_folder = "..\\gtk"
-print(test_folder)
+test_folder = "/usr/include"
 filelist = []
+folderlist = []
 processes = []
 threads = []
 
@@ -30,8 +31,16 @@ class Listener:
             data = self.signal_queue.get()
             print(data)
             # Check if finished                                                                       
-            if data[0]=="work_update":
+            if data[0] == "work_update":
                 print("Workers are working.")
+                self.signal_queue.task_done()
+                return
+            elif data[0] == "work_finished":
+                print("Workers are finished.")
+                self.signal_queue.task_done()
+                return
+            else:
+                self.signal_queue.task_done()
                 return
 
 class Worker:
@@ -40,14 +49,17 @@ class Worker:
         self.work_queue = wq
 
     def go(self):
+        #while self.work_queue.get(2) != None:
         while True:
-            print("Working")
+            #print("Working")
             task = self.work_queue.get(2)
-            print(task)
-            time.sleep(0.001)
+            #print(task)
+            time.sleep(0.01)
             self.work_queue.task_done()
+            print(self.work_queue.qsize())
             self.signal_queue.put("work_update")
         self.work_queue.task_done()
+        self.signal_queue.put("work_finished")
 
 class parser:
     def __init__(self, file):
@@ -59,9 +71,9 @@ class parser:
     def get_token(self, keyword):
         token = ""
         if keyword in self.prep:
-            token = self.prep(keyword)
+            token = self.prep.values(keyword)
         if keyword in self.reg:
-            token = self.reg(keyword)
+            token = self.reg.values(keyword)
         return token
 
     def parse_preprocess(self, token):
@@ -72,10 +84,10 @@ class parser:
         token = {}
         if word in self.prep:
             token.keys = word
-            token.values = self.prep(word)
+            token.values = self.prep.values(word)
         if word in self.reg:
             token.keys = word
-            token.values = self.reg(word)
+            token.values = self.reg.values(word)
         return token
 
     # Creates a list of elements per line in file,
@@ -85,7 +97,7 @@ class parser:
         word = [w for w in line.split()]
         for w in word:
             token = self.tokenize_word(w)
-            if token.value == 'PREPROCESS':
+            if token.values == 'PREPROCESS':
                 self.parse_preprocess(token)
         return tupline
 
@@ -106,8 +118,9 @@ def sourcedir_filecnt(sourcedir):
     return cnt
 
 def sourcedir_foldercnt(sourcedir):
-    ### Return the number of folders, if it contains '*.h' files, in sourcedir - including subdirectories ###
-    global cnt
+    # Return the number of folders, if it contains '*.h' files,
+    # in 'sourcedir' - including subdirectories.
+    cnt = 0
     global folderlist
     for folderName, subfolders, files in os.walk(sourcedir):
         if subfolders:
@@ -127,7 +140,7 @@ def process_files(gui, source, dest):
     global destdir
     sourcedir = source
     destdir = dest
-    pool = mp.Pool(processes=num_cores)
+    pool = multiprocessing.Pool(processes=num_cores)
     pool.map(process_file, filelist)
 
 def process_file(data):
@@ -175,11 +188,11 @@ def process_file(data):
     newfile = open(outputfile, "w")
     newfile.write(outfile)
     newfile.close()
-    
-test_folder = get_script_path()+'\\..\\gtk'
+
+
 print(test_folder)
 print('Number of *.h files in directory: ',sourcedir_filecnt(test_folder))
-print(num_cores)
+print('Number of processor cores: ',num_cores)
 
 print("Creating Listener")
 listener = Listener(signal_queue)
@@ -193,7 +206,7 @@ print(threads)
 print("Creating work queue")
 for i in filelist:
     work_queue.put(i)
-    print(work_queue.qsize())
+print(work_queue.qsize())
 
 for i in range(num_cores):
     w = Worker(signal_queue, work_queue)
@@ -203,6 +216,21 @@ for i in range(num_cores):
     processes.append(p)
 print(processes)
 
-work_queue.join()
 for i in range(num_cores):
-    work_queue.put(None)
+    work_queue.put(None)
+
+print('Items on work queue: ',work_queue.qsize())
+work_queue.join()
+print('Items on signal queue: ',signal_queue.qsize())
+signal_queue.join()
+
+print("Closing down workers")
+for p in processes:
+    print("Closing down:",p)
+    p.join()
+print(processes)
+
+print("Closing down Listener")
+for t in threads:
+    t.join()
+print(threads)