Commit dfa31851 authored by BO ZHANG's avatar BO ZHANG 🏀
Browse files

remove parallel in Dispatcher.dispatch_obsgroup_detector

parent b9d13950
...@@ -118,7 +118,7 @@ def extract_basis_table(dlist: list[dict], basis_keys: tuple) -> table.Table: ...@@ -118,7 +118,7 @@ def extract_basis_table(dlist: list[dict], basis_keys: tuple) -> table.Table:
def split_data_basis(data_basis: table.Table, n_split: int = 1) -> list[table.Table]: def split_data_basis(data_basis: table.Table, n_split: int = 1) -> list[table.Table]:
"""Split data basis into n_split parts.""" """Split data basis into n_split parts via obs_id"""
assert ( assert (
np.unique(data_basis["dataset"]).size == 1 np.unique(data_basis["dataset"]).size == 1
), "Only one dataset is allowed for splitting." ), "Only one dataset is allowed for splitting."
...@@ -546,16 +546,8 @@ class Dispatcher: ...@@ -546,16 +546,8 @@ class Dispatcher:
def dispatch_obsgroup_detector( def dispatch_obsgroup_detector(
plan_basis: table.Table, plan_basis: table.Table,
data_basis: table.Table, data_basis: table.Table,
n_jobs: int = 1, # n_jobs: int = 1,
): ):
# parallel
if n_jobs != 1:
task_list = joblib.Parallel(n_jobs=n_jobs)(
joblib.delayed(Dispatcher.dispatch_obsid)(plan_basis, _)
for _ in split_data_basis(data_basis, n_split=n_jobs)
)
return sum(task_list, [])
# unique obsgroup basis (using group_by) # unique obsgroup basis (using group_by)
obsgroup_basis = plan_basis.group_by( obsgroup_basis = plan_basis.group_by(
keys=[ keys=[
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment