Multithread with workerpool modul
import os
import urllib
import workerpool
from datetime import datetime
class DownloadJob(workerpool.Job):
def __init__(self, fa):
self.fa = fa
def run(self):
f = open(self.fa + '.txt','w')
f.write('Example Note.......')
f.close()
pool = workerpool.WorkerPool(size=5)
def workfile():
range1 = 51
range2 = 102
fam1 = 555
fam2 = 833
ranges = range2 -range1
fams = fam2 -fam1
workname = "Python"
path = os.getcwd()
os.system('mkdir ' + str(workname))
sTime = datetime.now()
for a in range(ranges + 1):
os.chdir(path + '\\' + str(workname))
os.system('mkdir ' + str(range1 + a))
os.chdir(path + '\\' + str(workname) + '\\' + str(range1 + a))
for b in range(fams + 1):
fa = str(fam1 + b)
job = DownloadJob(fa)
pool.put(job)
pool.shutdown()
pool.wait()
print 'Elapsed Time: %s' % (datetime.now() - sTime)
z = open('info.txt','w')
z.write('Elapsed Time: %s' % (datetime.now() - sTime))
z.close()
os.chdir(path + '\\' + str(workname))
tumSure = open('info.txt','w')
tumSure.write('Elapsed All Time: %s' % (datetime.now() - sTime))
tumSure.close()
print 'All Time: %s' % (datetime.now() - sTime)
print 'Workname : %s downloaded.' % (workname)
quit()
workfile()
Hi all,
I have a code as above and I want to use thread logic for creating file. Folder numbers start wird range1
, i.e. 51. Text files are created in this directory with the names 555.txt
to 833.txt
. But after creating the folder with name 52
it stops, failing to create 555.txt
to 833.txt
.
it think it stops because
pool.shutdown()
pool.wait()
How can I make the loop continue wit开发者_如何转开发h no stop?
I think you should only shutdown the pool if you are finished with it, i.e. after the for a
loop, maybe even in a try...finally
clause.
It would look that way:
try:
for a in range(ranges + 1):
os.chdir(path + '\\' + str(workname))
os.system('mkdir ' + str(range1 + a))
os.chdir(path + '\\' + str(workname) + '\\' + str(range1 + a))
for b in range(fams + 1):
fa = str(fam1 + b)
job = DownloadJob(fa)
pool.put(job)
finally:
pool.shutdown()
pool.wait()
In this way the pool
shutdown
a) happens only if all pool putting is done and b) happens even if there is an exception in order to cleanly shutdown.
If the pool had a context manager, it would be even simpler. But AFAICS, it hasn't. Otherwise, you could do
with pool:
for a in range(ranges + 1):
os.chdir(path + '\\' + str(workname))
os.system('mkdir ' + str(range1 + a))
os.chdir(path + '\\' + str(workname) + '\\' + str(range1 + a))
for b in range(fams + 1):
fa = str(fam1 + b)
job = DownloadJob(fa)
pool.put(job)
But if you want, you can do
from contextlib import contextmanager
@contextmanager
def shutdown_wait(pool):
try:
yield pool
finally:
pool.shutdown()
pool.wait()
...
with shutdown_wait(pool):
for a ... [as above]
精彩评论