### simple way to execure multiple process in parallel

2018-04-03

http://code.activestate.com/recipes/577376-simple-way-to-execute-multiple-process-in-parallel/

看起來是在同一時間利用多個PCU core 執行多個 importcenter 而不是把一個importcenter的工作拆給多個core 去運算 有點不太一樣..

不過用在 resetdata 上也許有用？

> 1.py

     import sys, os, time
     from subprocess import Popen, list2cmdline

     def cpu_count():
         ''' Returns the number of CPUs in the system
         '''
         num = 1
         if sys.platform == 'win32':
             try:
                 num = int(os.environ['NUMBER_OF_PROCESSORS'])
             except (ValueError, KeyError):
                 pass
         elif sys.platform == 'darwin':
             try:
                 num = int(os.popen('sysctl -n hw.ncpu').read())
             except ValueError:
                 pass
         else:
             try:
                 num = os.sysconf('SC_NPROCESSORS_ONLN')
             except (ValueError, OSError, AttributeError):
                 pass

         return num

     def exec_commands(cmds):
         ''' Exec commands in parallel in multiple process
         (as much as we have CPU)
         '''
         if not cmds: return # empty list

         def done(p):
             return p.poll() is not None
         def success(p):
             return p.returncode == 0
         def fail():
             sys.exit(1)

         max_task = cpu_count()
         processes = []
     import sys, os, time
     from subprocess import Popen, list2cmdline

     def cpu_count():
         ''' Returns the number of CPUs in the system
         '''
         num = 1
         if sys.platform == 'win32':
             try:
                 num = int(os.environ['NUMBER_OF_PROCESSORS'])
             except (ValueError, KeyError):
                 pass
         elif sys.platform == 'darwin':
             try:
                 num = int(os.popen('sysctl -n hw.ncpu').read())
             except ValueError:
                 pass
         else:
             try:
                 num = os.sysconf('SC_NPROCESSORS_ONLN')
             except (ValueError, OSError, AttributeError):
                 pass

         return num

     def exec_commands(cmds):
         ''' Exec commands in parallel in multiple process
         (as much as we have CPU)
         '''
         if not cmds: return # empty list

         def done(p):
             return p.poll() is not None
         def success(p):
             return p.returncode == 0
         def fail():
             sys.exit(1)

         max_task = cpu_count()
         processes = []

1.sh

    #!/bin/bash

    python manage.py importcenter A5088 fc-11167-01

2.sh

    #!/bin/bash

    python manage.py importcenter S5004 fc-20544-01

comments powered by Disqus
© 2017 by Lednerb
Bilberry Hugo Theme