patternpythonMinor
Semi-synchronous programming in Python
Viewed 0 times
programmingsemipythonsynchronous
Problem
I just took a stab at creating a library for something I'm dubbing semi-synchronous programming (async + dependencies). It's rather dense, but I'd really appreciate a quick code review to ensure I'm not introducing any race conditions. Feedback on best practices, general improvements is also welcome.
```
from multiprocessing import Queue, Process, Manager
from collections import defaultdict
from time import sleep
def queue_function(fn, args, kwargs):
semisync.q.put([fn(*args, **kwargs), id(fn)])
def start_process(fn, args, kwargs):
p = Process(target=queue_function, args=(fn, args, kwargs))
p.start()
semisync.processes.append(p)
def cleanup():
# ensure no processes remain in a zombie state
while semisync.processes:
p = semisync.processes.pop()
p.join()
def generate_dependency_trees(tree):
for fn in tree.keys():
for dependency in tree[fn].get('dependencies', []):
semisync.depends_on[fn].add(dependency)
semisync.needed_for[dependency].add(fn)
def dependencies(fn):
tree, completed = semisync.tree, semisync.completed
return [d for d in tree[fn]['dependencies'] if d not in completed]
def independent_fns(tree):
result = []
for key in tree.keys():
if not dependencies(key):
result.append(key)
return result
# wrap method in fn to call semisynchronously
def semisync_method(c, method_name):
def method(*args, **kwargs):
getattr(c, method_name)(*args, **kwargs)
return method
def merge_dicts(d1, d2):
for key in ['args', 'kwargs']:
d1[key] += d2.get(key, [])
return d1
class semisync:
tree = {}
q = Queue()
processes = []
map = {}
manager = Manager()
depends_on = defaultdict(set)
needed_for = defaultdict(set)
completed = set()
fn_map = {}
lock = manager.Lock()
def __init__(self, callback=False, dependencies=set()):
self.callback = callback
self.dependencies = dependencies
def __call__(self, fn):
"""Returns decorated function"""
def semisync_fn(*args, **kw
```
from multiprocessing import Queue, Process, Manager
from collections import defaultdict
from time import sleep
def queue_function(fn, args, kwargs):
semisync.q.put([fn(*args, **kwargs), id(fn)])
def start_process(fn, args, kwargs):
p = Process(target=queue_function, args=(fn, args, kwargs))
p.start()
semisync.processes.append(p)
def cleanup():
# ensure no processes remain in a zombie state
while semisync.processes:
p = semisync.processes.pop()
p.join()
def generate_dependency_trees(tree):
for fn in tree.keys():
for dependency in tree[fn].get('dependencies', []):
semisync.depends_on[fn].add(dependency)
semisync.needed_for[dependency].add(fn)
def dependencies(fn):
tree, completed = semisync.tree, semisync.completed
return [d for d in tree[fn]['dependencies'] if d not in completed]
def independent_fns(tree):
result = []
for key in tree.keys():
if not dependencies(key):
result.append(key)
return result
# wrap method in fn to call semisynchronously
def semisync_method(c, method_name):
def method(*args, **kwargs):
getattr(c, method_name)(*args, **kwargs)
return method
def merge_dicts(d1, d2):
for key in ['args', 'kwargs']:
d1[key] += d2.get(key, [])
return d1
class semisync:
tree = {}
q = Queue()
processes = []
map = {}
manager = Manager()
depends_on = defaultdict(set)
needed_for = defaultdict(set)
completed = set()
fn_map = {}
lock = manager.Lock()
def __init__(self, callback=False, dependencies=set()):
self.callback = callback
self.dependencies = dependencies
def __call__(self, fn):
"""Returns decorated function"""
def semisync_fn(*args, **kw
Solution
PEP0008 says to use 4 spaces per indentation level. Using just 2 can make it harder to see what is and isn't a full indentation.
You should use docstrings, especially since for
You should use docstrings, especially since for
cleanup you wrote a comment that could be one if it was formatted as a docstring. They're basically comments that are programmatically accessible so that someone else using your script can read what the function does.def cleanup():
"""Ensure no processes remain in a zombie state"""Code Snippets
def cleanup():
"""Ensure no processes remain in a zombie state"""Context
StackExchange Code Review Q#39315, answer score: 2
Revisions (0)
No revisions yet.