ligthwight: pool

#!/usr/bin/env python
import random
import multiprocessing
import os
 
def run(args):
    r1=args[0]
    r2=args[1]
    counter=args[2]
    com1="sleep %ss"%r1
    com2="sleep %ss"%r2
    print "a.) %i Executung %s"%(counter,com1)
    os.system(com1)
    print "b.) %i Executing %s"%(counter,com2)
    os.system(com2)
 
 
timelist=[]
for i in range(0,20):
    timelist.append([random.randint(1,5),random.randint(3,5),i+1])
 
pool = multiprocessing.Pool(processes = 8)
pool.map(run,timelist)
pool.close() # no more jobs accepted by pool
pool.join() # wait for jobs to finish

lightwight: subprocess

import random
 
commandlist=[]
for i in range(0,20):
    ti=random.randint(1,10)
    com="sleep %ss"%ti
    commandlist.append(com)
 
def submit_job_max_len(commandlist, max_processes):
    import subprocess
    import time
    sleep_time = 1.0
    processes = list()
    for command in commandlist:
        print 'running {n} processes. Submitting {proc}.'.format(n=len(processes),proc=str(command))
        processes.append(subprocess.Popen(command, shell=True, stdout=None))
        while len(processes) >= max_processes:
            time.sleep(sleep_time)
            processes = [proc for proc in processes if proc.poll() is None]
    while len(processes) > 0:
        time.sleep(sleep_time)
        processes = [proc for proc in processes if proc.poll() is None]
 
submit_job_max_len(commandlist, max_processes=8)
print "Done"

run tons of cmh-tests

#!/usr/bin/env python
import os
import sys
 
 
replicates=[1,3,5,10,15,20]
generations=[10,20,30,40,50,60]
cmhpath=sys.argv[1]
 
 
def generate_cmhcommand(cmhpath,input,output,rep,gen):
    samples=(10,20,30,40,50,60)
    samplehash={}
    c=1
    for s in samples:
        samplehash[s]=c
        c+=1
 
    baseoffset=len(samples)+1
    sampleoffset=samplehash[gen]
 
    tocomp=[]
    for i in range(0,rep):
        base=1+baseoffset*i
        der=base+sampleoffset
        tocomp.append("{0}-{1}".format(base,der))
    if rep==1:
        tocomp.append(tocomp[0])
 
    temppr=",".join(tocomp)
    topr="perl {0} --min-count 1 --min-coverage 1 --max-coverage 100000 --min-logpvalue 0.0 --population {1} --input {2} --output {3}".format(cmhpath,temppr,input, output)
    return topr
 
 
for r in replicates:
    for g in generations:
        dirname="cmh_r%s_g%s"%(r,g)
        if not os.path.exists(dirname):
            os.mkdir(dirname)
 
commandlist=[]
for r in replicates:
    for g in generations:
        dirname="cmh_r%s_g%s"%(r,g)
        for i in range(1,11):
            input="n%s.sync" % i
            output="%s/n%s.cmh"%(dirname,i)
            cmhcommand=generate_cmhcommand(cmhpath,input,output,r,g)
            commandlist.append(cmhcommand)
 
def submit_job_max_len(commandlist, max_processes):
    import subprocess
    import time
    sleep_time = 3.0
    processes = list()
    for command in commandlist:
        print 'running {n} processes. Submitting {proc}.'.format(n=len(processes),proc=str(command))
        processes.append(subprocess.Popen(command, shell=True, stdout=None))
        while len(processes) >= max_processes:
            time.sleep(sleep_time)
            processes = [proc for proc in processes if proc.poll() is None]
    while len(processes) > 0:
        time.sleep(sleep_time)
        processes = [proc for proc in processes if proc.poll() is None]
 
submit_job_max_len(commandlist, max_processes=8)
print "Done"