Add files via upload
This commit is contained in:
Lerking
GitHub
@@ -0,0 +1,34 @@
|
||||
import multiprocessing as mp
|
||||
import random
|
||||
import string
|
||||
|
||||
random.seed(123)
|
||||
|
||||
# Define an output queue
|
||||
output = mp.Queue()
|
||||
|
||||
# define a example function
|
||||
def rand_string(length, output):
|
||||
""" Generates a random string of numbers, lower- and uppercase chars. """
|
||||
rand_str = ''.join(random.choice(
|
||||
string.ascii_lowercase
|
||||
+ string.ascii_uppercase
|
||||
+ string.digits)
|
||||
for i in range(length))
|
||||
output.put(rand_str)
|
||||
|
||||
# Setup a list of processes that we want to run
|
||||
processes = [mp.Process(target=rand_string, args=(5, output)) for x in range(4)]
|
||||
|
||||
# Run processes
|
||||
for p in processes:
|
||||
p.start()
|
||||
|
||||
# Exit the completed processes
|
||||
for p in processes:
|
||||
p.join()
|
||||
|
||||
# Get process results from the output queue
|
||||
results = [output.get() for p in processes]
|
||||
|
||||
print(results)
|
||||
@@ -0,0 +1,163 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
GUI_WIN = 0
|
||||
GUI_LINUX = 1
|
||||
|
||||
import sys
|
||||
if sys.platform == 'win32':
|
||||
GUI = GUI_WIN
|
||||
else:
|
||||
GUI = GUI_LINUX
|
||||
|
||||
from tkinter import (Tk, BOTH, Text, E, W, S, N, END,
|
||||
NORMAL, DISABLED, StringVar)
|
||||
from tkinter.ttk import Frame, Label, Button, Progressbar, Entry
|
||||
from tkinter import scrolledtext
|
||||
|
||||
if GUI == GUI_WIN:
|
||||
from multiprocessing import Process, Manager, Queue
|
||||
from queue import Empty
|
||||
else:
|
||||
from multiprocessing import Queue, Process
|
||||
import queue
|
||||
|
||||
from decimal import Decimal, getcontext
|
||||
|
||||
DELAY1 = 80
|
||||
DELAY2 = 20
|
||||
|
||||
if GUI == GUI_WIN:
|
||||
# Queue must be global
|
||||
q = Queue()
|
||||
|
||||
class commonUI(Frame):
|
||||
def __init__(self, parent):
|
||||
Frame.__init__(self, parent)
|
||||
self.parent = parent
|
||||
self.initUI()
|
||||
|
||||
def initUI(self):
|
||||
|
||||
self.parent.title("Pi computation")
|
||||
self.pack(fill=BOTH, expand=True)
|
||||
|
||||
self.grid_columnconfigure(4, weight=1)
|
||||
self.grid_rowconfigure(3, weight=1)
|
||||
|
||||
lbl1 = Label(self, text="Digits:")
|
||||
lbl1.grid(row=0, column=0, sticky=E, padx=10, pady=10)
|
||||
|
||||
self.ent1 = Entry(self, width=10)
|
||||
self.ent1.insert(END, "4000")
|
||||
self.ent1.grid(row=0, column=1, sticky=W)
|
||||
|
||||
lbl2 = Label(self, text="Accuracy:")
|
||||
lbl2.grid(row=0, column=2, sticky=E, padx=10, pady=10)
|
||||
|
||||
self.ent2 = Entry(self, width=10)
|
||||
self.ent2.insert(END, "100")
|
||||
self.ent2.grid(row=0, column=3, sticky=W)
|
||||
|
||||
self.startBtn = Button(self, text="Start", command=self.onStart)
|
||||
self.startBtn.grid(row=1, column=0, padx=10, pady=5, sticky=W)
|
||||
|
||||
self.pbar = Progressbar(self, mode='indeterminate')
|
||||
self.pbar.grid(row=1, column=1, columnspan=3, sticky=W+E)
|
||||
|
||||
self.txt = scrolledtext.ScrolledText(self)
|
||||
self.txt.grid(row=2, column=0, rowspan=4, padx=10, pady=5, columnspan=5, sticky=E+W+S+N)
|
||||
|
||||
def onStart(self):
|
||||
|
||||
self.startBtn.config(state=DISABLED)
|
||||
self.txt.delete("1.0", END)
|
||||
|
||||
digits = int(self.ent1.get())
|
||||
accuracy = int(self.ent2.get())
|
||||
|
||||
self.p1 = Process(target=generatePi, args=(q, digits, accuracy))
|
||||
self.p1.start()
|
||||
self.pbar.start(DELAY2)
|
||||
self.after(DELAY1, self.onGetValue)
|
||||
|
||||
|
||||
def onGetValue(self):
|
||||
|
||||
if (self.p1.is_alive()):
|
||||
|
||||
self.after(DELAY1, self.onGetValue)
|
||||
return
|
||||
else:
|
||||
|
||||
try:
|
||||
|
||||
self.txt.insert('end', q.get(0))
|
||||
self.txt.insert('end', "\n")
|
||||
self.pbar.stop()
|
||||
self.startBtn.config(state=NORMAL)
|
||||
|
||||
except Empty:
|
||||
print("queue is empty")
|
||||
|
||||
class win_Example(Frame):
|
||||
|
||||
def __init__(self, parent):
|
||||
Frame.__init__(self, parent, name="frame")
|
||||
commonUI(parent)
|
||||
|
||||
if GUI == GUI_WIN:
|
||||
# Generate function must be a top-level module funtion
|
||||
def generatePi(q, digs, acc):
|
||||
|
||||
getcontext().prec = digs
|
||||
|
||||
pi = Decimal(0)
|
||||
k = 0
|
||||
n = acc
|
||||
|
||||
while k < n:
|
||||
pi += (Decimal(1)/(16**k))*((Decimal(4)/(8*k+1)) - \
|
||||
(Decimal(2)/(8*k+4)) - (Decimal(1)/(8*k+5))- \
|
||||
(Decimal(1)/(8*k+6)))
|
||||
k += 1
|
||||
|
||||
q.put(pi)
|
||||
|
||||
class linux_Example(Frame):
|
||||
|
||||
def __init__(self, parent, q):
|
||||
Frame.__init__(self, parent)
|
||||
|
||||
self.queue = q
|
||||
commonUI(parent)
|
||||
|
||||
def generatePi(self, queue):
|
||||
|
||||
getcontext().prec = self.digits
|
||||
|
||||
pi = Decimal(0)
|
||||
k = 0
|
||||
n = self.accuracy
|
||||
|
||||
while k < n:
|
||||
pi += (Decimal(1)/(16**k))*((Decimal(4)/(8*k+1)) - \
|
||||
(Decimal(2)/(8*k+4)) - (Decimal(1)/(8*k+5))- \
|
||||
(Decimal(1)/(8*k+6)))
|
||||
k += 1
|
||||
print (self.p1.is_alive())
|
||||
|
||||
queue.put(pi)
|
||||
print("end")
|
||||
|
||||
def main():
|
||||
if GUI == GUI_LINUX:
|
||||
q = Queue()
|
||||
|
||||
root = Tk()
|
||||
root.geometry("400x350+300+300")
|
||||
app = win_Example(root) if GUI == GUI_WIN else linux_Example(root)
|
||||
root.mainloop()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,133 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
"""
|
||||
ZetCode Tkinter e-book
|
||||
|
||||
This script produces a long-running task of calculating
|
||||
a large Pi number, while keeping the GUI responsive.
|
||||
|
||||
Author: Jan Bodnar
|
||||
Last modified: January 2016
|
||||
Website: www.zetcode.com
|
||||
"""
|
||||
|
||||
from tkinter import (Tk, BOTH, Text, E, W, S, N, END,
|
||||
NORMAL, DISABLED, StringVar)
|
||||
from tkinter.ttk import Frame, Label, Button, Progressbar, Entry
|
||||
from tkinter import scrolledtext
|
||||
|
||||
from multiprocessing import Queue, Process
|
||||
import queue
|
||||
from decimal import Decimal, getcontext
|
||||
|
||||
DELAY1 = 80
|
||||
DELAY2 = 20
|
||||
|
||||
class Example(Frame):
|
||||
|
||||
def __init__(self, parent, q):
|
||||
Frame.__init__(self, parent)
|
||||
|
||||
self.queue = q
|
||||
self.parent = parent
|
||||
self.initUI()
|
||||
|
||||
|
||||
def initUI(self):
|
||||
|
||||
self.parent.title("Pi computation")
|
||||
self.pack(fill=BOTH, expand=True)
|
||||
|
||||
self.grid_columnconfigure(4, weight=1)
|
||||
self.grid_rowconfigure(3, weight=1)
|
||||
|
||||
lbl1 = Label(self, text="Digits:")
|
||||
lbl1.grid(row=0, column=0, sticky=E, padx=10, pady=10)
|
||||
|
||||
self.ent1 = Entry(self, width=10)
|
||||
self.ent1.insert(END, "4000")
|
||||
self.ent1.grid(row=0, column=1, sticky=W)
|
||||
|
||||
lbl2 = Label(self, text="Accuracy:")
|
||||
lbl2.grid(row=0, column=2, sticky=E, padx=10, pady=10)
|
||||
|
||||
self.ent2 = Entry(self, width=10)
|
||||
self.ent2.insert(END, "100")
|
||||
self.ent2.grid(row=0, column=3, sticky=W)
|
||||
|
||||
self.startBtn = Button(self, text="Start",
|
||||
command=self.onStart)
|
||||
self.startBtn.grid(row=1, column=0, padx=10, pady=5, sticky=W)
|
||||
|
||||
self.pbar = Progressbar(self, mode='indeterminate')
|
||||
self.pbar.grid(row=1, column=1, columnspan=3, sticky=W+E)
|
||||
|
||||
self.txt = scrolledtext.ScrolledText(self)
|
||||
self.txt.grid(row=2, column=0, rowspan=4, padx=10, pady=5,
|
||||
columnspan=5, sticky=E+W+S+N)
|
||||
|
||||
|
||||
def onStart(self):
|
||||
|
||||
self.startBtn.config(state=DISABLED)
|
||||
self.txt.delete("1.0", END)
|
||||
|
||||
self.digits = int(self.ent1.get())
|
||||
self.accuracy = int(self.ent2.get())
|
||||
|
||||
self.p1 = Process(target=self.generatePi, args=(self.queue,))
|
||||
self.p1.start()
|
||||
self.pbar.start(DELAY2)
|
||||
self.after(DELAY1, self.onGetValue)
|
||||
|
||||
|
||||
def onGetValue(self):
|
||||
|
||||
if (self.p1.is_alive()):
|
||||
|
||||
self.after(DELAY1, self.onGetValue)
|
||||
return
|
||||
else:
|
||||
|
||||
try:
|
||||
self.txt.insert('end', self.queue.get(0))
|
||||
self.txt.insert('end', "\n")
|
||||
self.pbar.stop()
|
||||
self.startBtn.config(state=NORMAL)
|
||||
|
||||
except queue.Empty:
|
||||
print("queue is empty")
|
||||
|
||||
|
||||
def generatePi(self, queue):
|
||||
|
||||
getcontext().prec = self.digits
|
||||
|
||||
pi = Decimal(0)
|
||||
k = 0
|
||||
n = self.accuracy
|
||||
|
||||
while k < n:
|
||||
pi += (Decimal(1)/(16**k))*((Decimal(4)/(8*k+1)) - \
|
||||
(Decimal(2)/(8*k+4)) - (Decimal(1)/(8*k+5))- \
|
||||
(Decimal(1)/(8*k+6)))
|
||||
k += 1
|
||||
print (self.p1.is_alive())
|
||||
|
||||
queue.put(pi)
|
||||
print("end")
|
||||
|
||||
|
||||
def main():
|
||||
|
||||
q = Queue()
|
||||
|
||||
root = Tk()
|
||||
root.geometry("400x350+300+300")
|
||||
app = Example(root, q)
|
||||
root.mainloop()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,134 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
"""
|
||||
ZetCode Tkinter e-book
|
||||
|
||||
This script produces a long-running task of calculating
|
||||
a large Pi number, while keeping the GUI responsive.
|
||||
This is an example written for Windows.
|
||||
|
||||
Author: Jan Bodnar
|
||||
Last modified: January 2016
|
||||
Website: www.zetcode.com
|
||||
"""
|
||||
|
||||
from tkinter import (Tk, BOTH, Text, E, W, S, N, END,
|
||||
NORMAL, DISABLED, StringVar)
|
||||
from tkinter.ttk import Frame, Label, Button, Progressbar, Entry
|
||||
from tkinter import scrolledtext
|
||||
|
||||
from multiprocessing import Process, Manager, Queue
|
||||
from queue import Empty
|
||||
from decimal import Decimal, getcontext
|
||||
|
||||
|
||||
DELAY1 = 80
|
||||
DELAY2 = 20
|
||||
|
||||
# Queue must be global
|
||||
q = Queue()
|
||||
|
||||
class Example(Frame):
|
||||
|
||||
def __init__(self, parent):
|
||||
Frame.__init__(self, parent, name="frame")
|
||||
|
||||
self.parent = parent
|
||||
self.initUI()
|
||||
|
||||
|
||||
def initUI(self):
|
||||
|
||||
self.parent.title("Pi computation")
|
||||
self.pack(fill=BOTH, expand=True)
|
||||
|
||||
self.grid_columnconfigure(4, weight=1)
|
||||
self.grid_rowconfigure(3, weight=1)
|
||||
|
||||
lbl1 = Label(self, text="Digits:")
|
||||
lbl1.grid(row=0, column=0, sticky=E, padx=10, pady=10)
|
||||
|
||||
self.ent1 = Entry(self, width=10)
|
||||
self.ent1.insert(END, "4000")
|
||||
self.ent1.grid(row=0, column=1, sticky=W)
|
||||
|
||||
lbl2 = Label(self, text="Accuracy:")
|
||||
lbl2.grid(row=0, column=2, sticky=E, padx=10, pady=10)
|
||||
|
||||
self.ent2 = Entry(self, width=10)
|
||||
self.ent2.insert(END, "100")
|
||||
self.ent2.grid(row=0, column=3, sticky=W)
|
||||
|
||||
self.startBtn = Button(self, text="Start",
|
||||
command=self.onStart)
|
||||
self.startBtn.grid(row=1, column=0, padx=10, pady=5, sticky=W)
|
||||
|
||||
self.pbar = Progressbar(self, mode='indeterminate')
|
||||
self.pbar.grid(row=1, column=1, columnspan=3, sticky=W+E)
|
||||
|
||||
self.txt = scrolledtext.ScrolledText(self)
|
||||
self.txt.grid(row=2, column=0, rowspan=4, padx=10, pady=5,
|
||||
columnspan=5, sticky=E+W+S+N)
|
||||
|
||||
|
||||
def onStart(self):
|
||||
|
||||
self.startBtn.config(state=DISABLED)
|
||||
self.txt.delete("1.0", END)
|
||||
|
||||
digits = int(self.ent1.get())
|
||||
accuracy = int(self.ent2.get())
|
||||
|
||||
self.p1 = Process(target=generatePi, args=(q, digits, accuracy))
|
||||
self.p1.start()
|
||||
self.pbar.start(DELAY2)
|
||||
self.after(DELAY1, self.onGetValue)
|
||||
|
||||
|
||||
def onGetValue(self):
|
||||
|
||||
if (self.p1.is_alive()):
|
||||
|
||||
self.after(DELAY1, self.onGetValue)
|
||||
return
|
||||
else:
|
||||
|
||||
try:
|
||||
|
||||
self.txt.insert('end', q.get(0))
|
||||
self.txt.insert('end', "\n")
|
||||
self.pbar.stop()
|
||||
self.startBtn.config(state=NORMAL)
|
||||
|
||||
except Empty:
|
||||
print("queue is empty")
|
||||
|
||||
# Generate function must be a top-level module funtion
|
||||
def generatePi(q, digs, acc):
|
||||
|
||||
getcontext().prec = digs
|
||||
|
||||
pi = Decimal(0)
|
||||
k = 0
|
||||
n = acc
|
||||
|
||||
while k < n:
|
||||
pi += (Decimal(1)/(16**k))*((Decimal(4)/(8*k+1)) - \
|
||||
(Decimal(2)/(8*k+4)) - (Decimal(1)/(8*k+5))- \
|
||||
(Decimal(1)/(8*k+6)))
|
||||
k += 1
|
||||
|
||||
q.put(pi)
|
||||
|
||||
|
||||
def main():
|
||||
|
||||
root = Tk()
|
||||
root.geometry("400x350+300+300")
|
||||
app = Example(root)
|
||||
root.mainloop()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,43 @@
|
||||
# Test Code for Tkinter with threads
|
||||
import tkinter as Tk
|
||||
import multiprocessing
|
||||
from Queue import Empty, Full
|
||||
import time
|
||||
|
||||
class GuiApp(object):
|
||||
def __init__(self,q):
|
||||
self.root = Tk.Tk()
|
||||
self.root.geometry('300x100')
|
||||
self.text_wid = Tk.Text(self.root,height=100,width=100)
|
||||
self.text_wid.pack(expand=1,fill=Tk.BOTH)
|
||||
self.root.after(100,self.CheckQueuePoll,q)
|
||||
|
||||
def CheckQueuePoll(self,c_queue):
|
||||
try:
|
||||
str = c_queue.get(0)
|
||||
self.text_wid.insert('end',str)
|
||||
except Empty:
|
||||
pass
|
||||
finally:
|
||||
self.root.after(100, self.CheckQueuePoll, c_queue)
|
||||
|
||||
# Data Generator which will generate Data
|
||||
def GenerateData(q):
|
||||
for i in range(10):
|
||||
print("Generating Some Data, Iteration %s" %(i))
|
||||
time.sleep(2)
|
||||
q.put("Some Data from iteration %s \n" %(i))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
# Queue which will be used for storing Data
|
||||
|
||||
q = multiprocessing.Queue()
|
||||
q.cancel_join_thread() # or else thread that puts data will not term
|
||||
gui = GuiApp(q)
|
||||
t1 = multiprocessing.Process(target=GenerateData,args=(q,))
|
||||
t1.start()
|
||||
gui.root.mainloop()
|
||||
|
||||
t1.join()
|
||||
t2.join()
|
||||
@@ -0,0 +1,29 @@
|
||||
import time
|
||||
from multiprocessing import Process, Value, Lock
|
||||
|
||||
class Counter(object):
|
||||
def __init__(self, initval=0):
|
||||
self.val = Value('i', initval)
|
||||
self.lock = Lock()
|
||||
|
||||
def increment(self):
|
||||
with self.lock:
|
||||
self.val.value += 1
|
||||
|
||||
def value(self):
|
||||
with self.lock:
|
||||
return self.val.value
|
||||
|
||||
def func(counter):
|
||||
for i in range(500):
|
||||
time.sleep(0.001)
|
||||
counter.increment()
|
||||
|
||||
if __name__ == '__main__':
|
||||
counter = Counter(0)
|
||||
procs = [Process(target=func, args=(counter,)) for i in range(10)]
|
||||
|
||||
for p in procs: p.start()
|
||||
for p in procs: p.join()
|
||||
|
||||
print (counter.value())
|
||||
@@ -0,0 +1,40 @@
|
||||
import tkinter as tk
|
||||
from tkinter import ttk
|
||||
import tkinter.messagebox
|
||||
|
||||
import time # to simulate long-running function
|
||||
|
||||
from threading import Thread
|
||||
|
||||
class RedactionSolutions:
|
||||
|
||||
def __init__(self, master):
|
||||
tk.Button(master, text="Run RS Wizard", fg='green', command=self.run).grid(row=5)
|
||||
|
||||
def long_running_function(self):
|
||||
#self.working = tk.Label(root, text='In Progress. Please Wait...')
|
||||
self.working = ttk.Progressbar(root, orient='horizontal', mode='determinate')
|
||||
self.working.config(value=0, maximum=10)
|
||||
self.working.grid(row=6, columnspan=2)
|
||||
|
||||
for x in range(10):
|
||||
print(x)
|
||||
self.working.config(value=x)
|
||||
time.sleep(1)
|
||||
|
||||
restart = tkinter.messagebox.askquestion('RS Wizard', 'Redaction complete! See file location for result.\nWould you like to redact another file?')
|
||||
if restart == 'yes':
|
||||
#self.reset()
|
||||
self.working.grid_forget()
|
||||
else:
|
||||
root.destroy()
|
||||
|
||||
def run(self):
|
||||
Thread(target=self.long_running_function).start()
|
||||
|
||||
|
||||
|
||||
root = tk.Tk()
|
||||
root.wm_title("RS Wizard")
|
||||
RedactionSolutions(root)
|
||||
root.mainloop()
|
||||
@@ -0,0 +1 @@
|
||||
/data_2/Projects/h2inc/multiprocessing_test.py
|
||||
@@ -0,0 +1,314 @@
|
||||
#!/usr/bin/env python3.5
|
||||
|
||||
# You are free to use and/or change this code for
|
||||
# your own needs.
|
||||
|
||||
# Original code (c)2018 Jan Lerking
|
||||
# Program to test various scenarios of
|
||||
# single thread, multi thread, pool and process
|
||||
|
||||
import os
|
||||
import sys
|
||||
import multiprocessing as mp
|
||||
import time
|
||||
import io
|
||||
import cpuinfo
|
||||
import queue
|
||||
import threading
|
||||
import time
|
||||
import random
|
||||
import string
|
||||
import shutil
|
||||
from tkinter import Tk, ttk, Label, Button, LabelFrame
|
||||
from tkinter import filedialog, Entry, Checkbutton
|
||||
from tkinter import Grid, StringVar, DoubleVar
|
||||
from tkinter import N, E, S, W
|
||||
from tkinter import DISABLED, NORMAL
|
||||
from os import errno
|
||||
import os
|
||||
from os.path import expanduser
|
||||
import platform
|
||||
class file_progress:
|
||||
def __init__(self, master):
|
||||
self.sourcedir = StringVar()
|
||||
self.destdir = StringVar()
|
||||
self.addinc = StringVar()
|
||||
self.cfilevar = DoubleVar()
|
||||
self.totalvar = 0
|
||||
self.filecnt = 0
|
||||
self.infofolder = 'Number of folders: 0'
|
||||
self.infofile = 'Number of headers: 0'
|
||||
self.currentfile = 'Current file: '
|
||||
self.totprogress = 'Total progress: '
|
||||
self.sourcedir.set('Select source directory!')
|
||||
self.destdir.set('Select destination directory!')
|
||||
|
||||
self.master = master
|
||||
self.master.title('File progress')
|
||||
self.master.grid_columnconfigure(1, weight=1)
|
||||
|
||||
self.frame = LabelFrame(master, text='Progress')
|
||||
self.frame.grid(row=0, column=0, columnspan=3, sticky=N+S+E+W, padx=5, pady=5)
|
||||
self.frame.grid_columnconfigure(1, weight=1)
|
||||
|
||||
self.totallabel = Label(self.frame, text=self.totprogress)
|
||||
self.totallabel.grid(row=8, column=0, sticky=W, padx=5, pady=5)
|
||||
self.totallabel.config(state=DISABLED)
|
||||
|
||||
self.totalprogress = ttk.Progressbar(self.frame, orient='horizontal', mode='determinate')
|
||||
self.totalprogress.grid(row=9, column=0, columnspan=3, sticky=N+S+E+W, padx=5, pady=5)
|
||||
self.totalprogress.config(variable=self.totalvar, maximum=self.filecnt)
|
||||
|
||||
def select_sourcedir(self, sourcedir):
|
||||
root.directory = os.path.abspath(filedialog.askdirectory())
|
||||
if root.directory:
|
||||
sourcedir.set(root.directory)
|
||||
filecnt = sourcedir_filecnt(root.directory)
|
||||
if filecnt > 0:
|
||||
tempstr = 'Number of headers: '+str(filecnt)
|
||||
temptot = 'Total progress: 0 of '+str(filecnt)
|
||||
print ('Source directory: ', sourcedir.get())
|
||||
self.destlabel.config(state=NORMAL)
|
||||
self.destentry.config(state=NORMAL)
|
||||
self.destdir_button.config(state=NORMAL)
|
||||
self.infofiles.config(text=tempstr)
|
||||
self.totallabel.config(text=temptot)
|
||||
self.filecnt = filecnt
|
||||
foldercnt = sourcedir_foldercnt(root.directory)
|
||||
if foldercnt > 0:
|
||||
tempstr = 'Number of folders: '+str(foldercnt)
|
||||
self.infofolders.config(text=tempstr)
|
||||
|
||||
def select_destdir(self, destdir):
|
||||
root.directory = filedialog.askdirectory()
|
||||
if root.directory:
|
||||
destdir.set(root.directory)
|
||||
print ('Destination directory: ', destdir.get())
|
||||
self.incchkbox.config(state=NORMAL)
|
||||
self.infofolders.config(state=NORMAL)
|
||||
self.infofiles.config(state=NORMAL)
|
||||
self.translate_button.config(state=NORMAL)
|
||||
self.cfilelabel.config(state=NORMAL)
|
||||
self.totallabel.config(state=NORMAL)
|
||||
|
||||
def translate(self, destdir, sourcedir, addinc):
|
||||
doinc = addinc.get()
|
||||
dest = destdir.get()
|
||||
source = sourcedir.get()
|
||||
if doinc == 'yes':
|
||||
dest = dest+'/include'
|
||||
print(os.path.exists(os.path.dirname(dest)))
|
||||
if not os.path.exists(os.path.dirname(dest)):
|
||||
try:
|
||||
os.makedirs(os.path.dirname(dest))
|
||||
except OSError as exc: # Guard against race condition
|
||||
if exc.errno != errno.EEXIST:
|
||||
raise
|
||||
destdir.set(dest)
|
||||
print ('Destination directory: ', destdir.get())
|
||||
process_files(source, dest)
|
||||
|
||||
def cfileprogress_update(cnt):
|
||||
self.cfilevar = cnt
|
||||
|
||||
def currentfile_update(current):
|
||||
self.currentfile = 'Current file: '+current
|
||||
|
||||
root = Tk()
|
||||
root.update()
|
||||
#root.minsize(350, 210)
|
||||
#width = (root.winfo_screenwidth()/2)-(350/2)
|
||||
#height = (root.winfo_screenheight()/2)-(210/2)
|
||||
#root.geometry('+%d+%d' % (width, height))
|
||||
root.resizable(False, False)
|
||||
f_progress = file_progress(root)
|
||||
root.mainloop()
|
||||
|
||||
cpu_info = cpuinfo.get_cpu_info()
|
||||
num_cores = mp.cpu_count()
|
||||
print(num_cores)
|
||||
|
||||
filelist = []
|
||||
exitFlag = 0
|
||||
queueLock = threading.Lock()
|
||||
workQueue = queue.Queue(0)
|
||||
threads = []
|
||||
class myThread (threading.Thread):
|
||||
def __init__(self, threadID, name, q):
|
||||
threading.Thread.__init__(self)
|
||||
self.threadID = threadID
|
||||
self.name = name
|
||||
self.q = q
|
||||
def run(self):
|
||||
print ("Starting " + self.name)
|
||||
process_data(self.name, self.q)
|
||||
print ("Exiting " + self.name)
|
||||
|
||||
def process_data(threadName, q):
|
||||
while not exitFlag:
|
||||
queueLock.acquire()
|
||||
if not workQueue.empty():
|
||||
data = q.get()
|
||||
process_file(data)
|
||||
queueLock.release()
|
||||
print ("%s processing %s" % (threadName, data))
|
||||
else:
|
||||
queueLock.release()
|
||||
time.sleep(1)
|
||||
|
||||
def sourcedir_filecnt(sourcedir):
|
||||
### Return the number of files, ending with '.h', in sourcedir - including subdirectories ###
|
||||
cnt = 0
|
||||
global filelist
|
||||
for folderName, subfolders, files in os.walk(sourcedir):
|
||||
for file in files:
|
||||
if file.lower().endswith('.h'):
|
||||
cnt += 1
|
||||
filelist += [folderName+'/'+file]
|
||||
return cnt
|
||||
|
||||
def process_files(sourcedir, destdir):
|
||||
global filelist
|
||||
for f in filelist:
|
||||
process_file(f)
|
||||
|
||||
def process_file(data):
|
||||
outfile = ''
|
||||
inputfile = data
|
||||
encodings = ['utf-8', 'latin-1', 'windows-1250', 'windows-1252', 'ascii',
|
||||
'big5', 'big5hkscs', 'cp037', 'cp273', 'cp424', 'cp437', 'cp500',
|
||||
'cp720', 'cp737', 'cp775', 'cp850', 'cp852', 'cp855', 'cp856',
|
||||
'cp857', 'cp858', 'cp860', 'cp861', 'cp862', 'cp863', 'cp864', 'cp865',
|
||||
'cp866', 'cp869', 'cp874', 'cp875', 'cp932', 'cp949', 'cp950', 'cp1006',
|
||||
'cp1026', 'cp1125', 'cp1140', 'cp1250', 'cp1251', 'cp1252', 'cp1253', 'cp1254',
|
||||
'cp1255', 'cp1256', 'cp1257', 'cp1258', 'cp65001', 'euc-jp', 'euc-jis-2004',
|
||||
'euc-jisx0213', 'euc-kr', 'gb2312', 'gbk', 'gb18030', 'hz', 'iso2022-jp',
|
||||
'iso2022-jp-1', 'iso2022-jp-2', 'iso2022-jp-2004', 'iso2022-jp-3',
|
||||
'iso2022-jp-ext', 'iso2022-kr', 'iso8859-2', 'iso8859-3', 'iso8859-4',
|
||||
'iso8859-5', 'iso8859-6', 'iso8859-7', 'iso8859-8', 'iso8859-9', 'iso8859-10',
|
||||
'iso8859-11', 'iso8859-13', 'iso8859-14', 'iso8859-15', 'iso8859-16', 'johab',
|
||||
'koi8-r', 'koi8-t', 'koi8-u', 'kz1048', 'mac-cyrillic', 'mac-greek',
|
||||
'mac-iceland', 'mac-latin2', 'mac-roman', 'mac-turkish', 'ptcp154',
|
||||
'shift-jis', 'shift-jis-2004', 'shift-jisx0213', 'utf-32', 'utf-32-be',
|
||||
'utf-32-le', 'utf-16', 'utf-16-be', 'utf-16-le', 'utf-7', 'utf-8-sig']
|
||||
for e in encodings:
|
||||
try:
|
||||
fh = io.open(data, 'r', encoding=e)
|
||||
fh.readlines()
|
||||
fh.seek(0)
|
||||
except UnicodeDecodeError:
|
||||
print('got unicode error with %s , trying different encoding' % e)
|
||||
else:
|
||||
#print('opening the file with encoding: %s ' % e)
|
||||
break
|
||||
#print(os.path.basename(data))
|
||||
for lines in fh:
|
||||
outfile = outfile+lines
|
||||
fh.close()
|
||||
outputfile = os.path.splitext(inputfile)[0]+'.inc'
|
||||
outputfile = str(outputfile).replace(sourcedir, destdir)
|
||||
#print(outputfile)
|
||||
if not os.path.exists(os.path.dirname(outputfile)):
|
||||
try:
|
||||
os.makedirs(os.path.dirname(outputfile))
|
||||
except OSError as exc: # Guard against race condition
|
||||
if exc.errno != errno.EEXIST:
|
||||
raise
|
||||
newfile = open(outputfile, "w")
|
||||
newfile.write(outfile)
|
||||
newfile.close()
|
||||
|
||||
def single_thread():
|
||||
t1 = time.time()
|
||||
cnt = sourcedir_filecnt(sourcedir)
|
||||
print('Files in: '+str(cnt))
|
||||
process_files(sourcedir, destdir)
|
||||
print('Single thread process time: '+str(time.time()-t1))
|
||||
|
||||
def multi_thread():
|
||||
global filelist
|
||||
global exitFlag
|
||||
|
||||
t1 = time.time()
|
||||
cnt = sourcedir_filecnt(sourcedir)
|
||||
threadList = ["thread 1", "thread 2", "thread 3", "thread 4", "thread 5"]
|
||||
nameList = filelist
|
||||
threadID = 0
|
||||
|
||||
# Create new threads
|
||||
for tName in threadList:
|
||||
thread = myThread(threadID, tName, workQueue)
|
||||
thread.start()
|
||||
threads.append(thread)
|
||||
threadID += 1
|
||||
|
||||
# Fill the queue
|
||||
queueLock.acquire()
|
||||
for word in nameList:
|
||||
workQueue.put(word)
|
||||
queueLock.release()
|
||||
|
||||
# Wait for queue to empty
|
||||
while not workQueue.empty():
|
||||
pass
|
||||
|
||||
# Notify threads it's time to exit
|
||||
exitFlag = 1
|
||||
|
||||
# Wait for all threads to complete
|
||||
for t in threads:
|
||||
t.join()
|
||||
print ("Exiting Main Thread")
|
||||
print('Files in: '+str(cnt))
|
||||
print('Multi thread process time: '+str(time.time()-t1))
|
||||
|
||||
def async_process(num):
|
||||
t1 = time.time()
|
||||
cnt = sourcedir_filecnt(sourcedir)
|
||||
|
||||
pool = mp.Pool(processes=num)
|
||||
#[pool.apply_async(process_file, args=(f,)) for f in filelist]
|
||||
pool.map(process_file, filelist)
|
||||
#output = [p.get() for p in results]
|
||||
#print(output)
|
||||
|
||||
# Setup a list of processes that we want to run
|
||||
#processes = [mp.Process(target=rand_string, args=(5, output)) for x in range(8)]
|
||||
#processes = [mp.Process(target=process_files, args=(sourcedir, destdir)) for x in range(num)]
|
||||
|
||||
# Run processes
|
||||
#for p in processes:
|
||||
#p.start()
|
||||
|
||||
# Exit the completed processes
|
||||
#for p in processes:
|
||||
#p.join()
|
||||
|
||||
# Get process results from the output queue
|
||||
#results = [output.get() for p in processes]
|
||||
|
||||
#print(results)
|
||||
print('Files in: '+str(cnt))
|
||||
print('Multi ('+str(num)+') processes time: '+str(time.time()-t1))
|
||||
|
||||
def cleanup(dest):
|
||||
shutil.rmtree(dest)
|
||||
|
||||
#sourcedir = 'C:/Users/dksojlg/Documents/gtk+-3.22.26'
|
||||
sourcedir = '/usr/include'
|
||||
#destdir = 'C:/Users/dksojlg/Documents/include'
|
||||
destdir = '/data_2/include'
|
||||
|
||||
#print(cpu_info)
|
||||
#single_thread() #2543 files - Single thread process time: 1.3732633590698242 sec.
|
||||
#os.rmdir(destdir)
|
||||
#multi_thread() #2543 files - Syncronized 5 threads process time: 80.25179100036621 sec.
|
||||
#os.rmdir(destdir)
|
||||
async_process(16) #2543 files - 2 processes, process time: 1.2379400730133057 sec.
|
||||
#cleanup(destdir)
|
||||
#async_process(4) #2543 files - 2 processes, process time: 2.6622860431671143 sec.
|
||||
#cleanup(destdir)
|
||||
#async_process(6) #2543 files - 2 processes, process time: 2.6622860431671143 sec.
|
||||
#cleanup(destdir)
|
||||
#async_process(8) #2543 files - 2 processes, process time: 2.6622860431671143 sec.
|
||||
#cleanup(destdir)
|
||||
@@ -0,0 +1,20 @@
|
||||
from multiprocessing import Pool
|
||||
|
||||
import time
|
||||
|
||||
work = (["A", 5], ["B", 2], ["C", 1], ["D", 3])
|
||||
|
||||
|
||||
def work_log(work_data):
|
||||
print(" Process %s waiting %s seconds" % (work_data[0], work_data[1]))
|
||||
time.sleep(int(work_data[1]))
|
||||
print(" Process %s Finished." % work_data[0])
|
||||
|
||||
|
||||
def pool_handler():
|
||||
p = Pool(2)
|
||||
p.map(work_log, work)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
pool_handler()
|
||||
@@ -0,0 +1,59 @@
|
||||
from Queue import Queue
|
||||
import threading
|
||||
import time
|
||||
|
||||
exitFlag = 0
|
||||
|
||||
class myThread (threading.Thread):
|
||||
def __init__(self, threadID, name, q):
|
||||
threading.Thread.__init__(self)
|
||||
self.threadID = threadID
|
||||
self.name = name
|
||||
self.q = q
|
||||
def run(self):
|
||||
print ("Starting " + self.name)
|
||||
process_data(self.name, self.q)
|
||||
print ("Exiting " + self.name)
|
||||
|
||||
def process_data(threadName, q):
|
||||
while not exitFlag:
|
||||
queueLock.acquire()
|
||||
if not workQueue.empty():
|
||||
data = q.get()
|
||||
queueLock.release()
|
||||
print ("%s processing %s" % (threadName, data))
|
||||
else:
|
||||
queueLock.release()
|
||||
time.sleep(1)
|
||||
|
||||
threadList = ["Thread-1", "Thread-2", "Thread-3"]
|
||||
nameList = ["One", "Two", "Three", "Four", "Five"]
|
||||
queueLock = threading.Lock()
|
||||
workQueue = Queue.Queue(10)
|
||||
threads = []
|
||||
threadID = 1
|
||||
|
||||
# Create new threads
|
||||
for tName in threadList:
|
||||
thread = myThread(threadID, tName, workQueue)
|
||||
thread.start()
|
||||
threads.append(thread)
|
||||
threadID += 1
|
||||
|
||||
# Fill the queue
|
||||
queueLock.acquire()
|
||||
for word in nameList:
|
||||
workQueue.put(word)
|
||||
queueLock.release()
|
||||
|
||||
# Wait for queue to empty
|
||||
while not workQueue.empty():
|
||||
pass
|
||||
|
||||
# Notify threads it's time to exit
|
||||
exitFlag = 1
|
||||
|
||||
# Wait for all threads to complete
|
||||
for t in threads:
|
||||
t.join()
|
||||
print ("Exiting Main Thread")
|
||||
@@ -0,0 +1,57 @@
|
||||
from multiprocessing import Process, Value, Lock
|
||||
from tkinter import Tk
|
||||
from tkinter import Tk, ttk, Label, Button, LabelFrame
|
||||
from tkinter import Grid, StringVar, DoubleVar
|
||||
from tkinter import N, E, S, W
|
||||
from tkinter import DISABLED, NORMAL
|
||||
import os
|
||||
import sys
|
||||
|
||||
dirname = 'C:\\Users\\dksojlg\\Documents'
|
||||
|
||||
platform = sys.platform
|
||||
print(platform)
|
||||
|
||||
class testGUI:
|
||||
def __init__(self, master, initval=0):
|
||||
self.progress = Value('i', initval)
|
||||
self.dircount = Value('i', initval)
|
||||
self.foldercnt = 'Folders: {}'.format(self.dircount.value)
|
||||
self.lock = Lock()
|
||||
|
||||
self.master = master
|
||||
self.master.title('Multiprocess progressbar test!')
|
||||
self.master.grid_columnconfigure(1, weight=1)
|
||||
|
||||
self.frame = LabelFrame(master, text='Folder count')
|
||||
self.frame.grid(row=0, column=0, columnspan=3, sticky=N+S+E+W, padx=5, pady=5)
|
||||
self.frame.grid_columnconfigure(1, weight=1)
|
||||
|
||||
self.sourcelabel = Label(self.frame, text=self.foldercnt)
|
||||
self.sourcelabel.grid(row=0, column=0, sticky=E, padx=(5, 1), pady=5)
|
||||
|
||||
self.sourcedir_button = Button(self.frame, text="Count folders...", command= lambda: get_directories(self))
|
||||
self.sourcedir_button.grid(row=0, column=2, sticky=W, padx=(3, 5), pady=5)
|
||||
|
||||
def increment_dir(self):
|
||||
with self.lock:
|
||||
self.dircount.value += 1
|
||||
self.sourcelabel.config(text=self.foldercnt)
|
||||
|
||||
def read_dir(testgui, dirname):
|
||||
for folderName, subfolders, files in os.walk(dirname):
|
||||
if subfolders:
|
||||
for subfolder in subfolders:
|
||||
read_dir(testgui, subfolder)
|
||||
#testgui.increment_dir()
|
||||
|
||||
def get_directories(testgui):
|
||||
p = Process(target=read_dir, args=(testgui, dirname,))
|
||||
p.start()
|
||||
|
||||
if __name__ == "__main__":
|
||||
root = Tk()
|
||||
root.update()
|
||||
root.resizable(False, False)
|
||||
testgui = testGUI(root)
|
||||
root.mainloop()
|
||||
Reference in New Issue
Block a user