update dispatcher.py (df44de32) · Commits · csst-cicd / csst-dag

csst_dag/_csst/dict.py

+1 −1

Original line number	Diff line number	Diff line
		@@ -9,7 +9,7 @@ class DotDict(dict):
		def __getattr__(self, key):
		"""属性访问优先级：1. 内置属性 → 2. 键值 → 3. 报错"""
		try:
		# 优先返回内置属性（如 keys, items 等方法）
		# 优先返回内置属性（如 basis_keys, items 等方法）
		return object.__getattribute__(self, key)
		except AttributeError:
		if key in self:

csst_dag/_csst/instrument.py

+5 −0

Original line number	Diff line number	Diff line
		@@ -137,6 +137,11 @@ class Telescope(DotDict):
		def n_instrument(self):
		return len(self.instruments)

		# def plan_to_detector(self, plan_data):
		# # convert to dict
		# plan_dict = dict(plan_data)
		# if plan_dict["instrument"] == "HSTDM":


		mbi = SimpleInstrument(
		name="MBI",

csst_dag/dag/init.py

+5 −4

Original line number	Diff line number	Diff line
		from ._base_dag import BaseDAG
		from ._dag_list import DAG_LIST
		from .dags import GeneralDAGViaObsid
		from .dags import GeneralDAGViaObsid, GeneralDAGViaObsgroup
		from .dispatcher import Dispatcher


		class CsstDAGs(dict):
		@@ -21,9 +22,9 @@ class CsstDAGs(dict):
		"csst-msc-l1-sls": GeneralDAGViaObsid(
		dag_group="msc-l1", dag="csst-msc-l1-sls", use_detector=True
		),
		# "csst-msc-l1-ooc": GeneralDAGViaObsgroup(
		# dag_group="msc-l1", dag="csst-msc-l1-ooc"
		# ),
		"csst-msc-l1-ooc": GeneralDAGViaObsgroup(
		dag_group="msc-l1-ooc", dag="csst-msc-l1-ooc"
		),
		"csst-cpic-l1": GeneralDAGViaObsid(
		dag_group="cpic-l1", dag="csst-cpic-l1", use_detector=True
		),

csst_dag/dag/dispatcher.py

+75 −58

Original line number	Diff line number	Diff line
		@@ -128,6 +128,9 @@ DATA_BASIS_KEYS = (
		"_id",
		)

		# join_type for data x plan
		PLAN_JOIN_TYPE = "inner"


		class Dispatcher:
		"""
		@@ -188,8 +191,8 @@ class Dispatcher:
		plan_basis: table.Table,
		data_basis: table.Table,
		) -> list[dict]:
		# unique obsid
		u_obsid = table.unique(data_basis["dataset", "obs_id"])
		# unique obsid --> useless
		# u_obsid = table.unique(data_basis["dataset", "obs_id"])

		# initialize task list
		task_list = []
		@@ -201,9 +204,10 @@ class Dispatcher:
		dynamic_ncols=True,
		):
		# i_data_basis = 1
		this_task = dict(data_basis[i_data_basis])
		this_data_basis = data_basis[i_data_basis : i_data_basis + 1]
		this_relevant_plan = table.join(
		u_obsid,
		this_data_basis,
		plan_basis,
		keys=["dataset", "obs_id"],
		join_type="inner",
		@@ -211,10 +215,12 @@ class Dispatcher:
		# append this task
		task_list.append(
		dict(
		task=this_data_basis,
		task=this_task,
		success=True,
		relevant_plan=this_relevant_plan,
		relevant_data=data_basis[i_data_basis : i_data_basis + 1],
		n_relevant_plan=len(this_relevant_plan),
		n_relevant_data=1,
		)
		)

		@@ -251,7 +257,7 @@ class Dispatcher:
		u_obsid,
		plan_basis,
		keys=["dataset", "obs_id"],
		join_type="left",
		join_type=PLAN_JOIN_TYPE,
		)
		print(f"{len(relevant_plan)} relevant plan records")

		@@ -303,7 +309,7 @@ class Dispatcher:
		"obs_group",
		"obs_id",
		],
		join_type="left",
		join_type=PLAN_JOIN_TYPE,
		)

		# whether detector effective
		@@ -312,7 +318,12 @@ class Dispatcher:
		this_detector_effective = (
		this_detector in csst[this_instrument].effective_detector_names
		)
		n_files_expected = this_data_detector_plan["n_frame"][0]

		n_files_expected = (
		this_data_detector_plan["n_frame"][0]
		if len(this_data_detector_plan) > 0
		else 0
		)
		n_files_found = len(this_data_detector_files)
		# append this task
		task_list.append(
		@@ -326,6 +337,8 @@ class Dispatcher:
		),
		relevant_plan=this_data_detector_plan,
		relevant_data=this_data_detector_files,
		n_relevant_plan=len(this_data_detector_plan),
		n_relevant_data=len(this_data_detector_files),
		)
		)
		return task_list
		@@ -350,7 +363,7 @@ class Dispatcher:
		u_obsid,
		plan_basis,
		keys=["dataset", "obs_id"],
		join_type="left",
		join_type=PLAN_JOIN_TYPE,
		)
		print(f"{len(relevant_plan)} relevant plan records")

		@@ -373,12 +386,12 @@ class Dispatcher:
		unit="task",
		dynamic_ncols=True,
		):
		i_data_obsid = 2
		# i_data_obsid = 2
		this_task = dict(u_data_obsid[i_data_obsid])
		this_data_obsid = u_data_obsid[i_data_obsid : i_data_obsid + 1]

		# join data and plan
		this_data_obsid_files = table.join(
		this_data_obsid_file = table.join(
		this_data_obsid,
		data_basis,
		keys=[
		@@ -400,13 +413,32 @@ class Dispatcher:
		"obs_group",
		"obs_id",
		],
		join_type="left",
		join_type=PLAN_JOIN_TYPE,
		)

		# whether effective detectors all there
		this_instrument = this_data_obsid["instrument"][0]
		this_success = set(csst[this_instrument].effective_detector_names).issubset(
		set(this_data_obsid_files["detector"])
		this_n_frame = (
		this_data_obsid_plan["n_frame"] if len(this_data_obsid_plan) > 0 else 0
		)
		this_effective_detector_names = csst[
		this_instrument
		].effective_detector_names

		if this_instrument == "HSTDM":
		# 不确定以后是1个探测器还是2个探测器
		this_n_file_found = len(this_data_obsid_file)
		this_n_file_expected = (this_n_frame, this_n_frame * 2)
		this_success = this_n_file_found in this_n_file_expected
		else:
		# for other instruments, e.g., MSC
		# n_file_found = len(this_obsgroup_obsid_file)
		# n_file_expected = len(effective_detector_names)
		# this_success &= n_file_found == n_file_expected

		# or more strictly, expected files are a subset of files found
		this_success = set(this_effective_detector_names) <= set(
		this_data_obsid_file["detector"]
		)

		# append this task
		@@ -415,7 +447,9 @@ class Dispatcher:
		task=this_task,
		success=this_success,
		relevant_plan=this_data_obsid_plan,
		relevant_data=this_data_obsid_files,
		relevant_data=this_data_obsid_file,
		n_relevant_plan=len(this_data_obsid_plan),
		n_relevant_data=len(this_data_obsid_file),
		)
		)

		@@ -452,14 +486,14 @@ class Dispatcher:
		this_task = dict(obsgroup_basis[i_obsgroup])
		this_success = True

		this_obsgroup_obsid = table.join(
		this_obsgroup_plan = table.join(
		obsgroup_basis[i_obsgroup : i_obsgroup + 1], # this obsgroup
		plan_basis,
		keys=["dataset", "instrument", "obs_type", "obs_group"],
		join_type="left",
		join_type=PLAN_JOIN_TYPE,
		)
		this_obsgroup_file = table.join(
		this_obsgroup_obsid,
		this_obsgroup_plan,
		data_basis,
		keys=["dataset", "instrument", "obs_type", "obs_group", "obs_id"],
		join_type="inner",
		@@ -467,36 +501,37 @@ class Dispatcher:
		)

		# loop over obsid
		for i_obsid in range(len(this_obsgroup_obsid)):
		for i_obsid in range(len(this_obsgroup_plan)):
		# i_obsid = 1
		# print(i_obsid)
		instrument = this_obsgroup_obsid[i_obsid]["instrument"]
		n_frame = this_obsgroup_obsid[i_obsid]["n_frame"]
		effective_detector_names = csst[instrument].effective_detector_names
		this_instrument = this_obsgroup_plan[i_obsid]["instrument"]
		this_n_frame = this_obsgroup_plan[i_obsid]["n_frame"]
		this_effective_detector_names = csst[
		this_instrument
		].effective_detector_names

		this_obsgroup_obsid_file = table.join(
		this_obsgroup_obsid[i_obsid : i_obsid + 1], # this obsid
		this_obsgroup_plan[i_obsid : i_obsid + 1], # this obsid
		data_basis,
		keys=["dataset", "instrument", "obs_type", "obs_group", "obs_id"],
		join_type="inner",
		table_names=["plan", "data"],
		)

		if instrument == "HSTDM": # 我也不知道太赫兹要怎么玩
		# this_success &= (
		# len(this_obsgroup_obsid_file) == n_frame
		# or len(this_obsgroup_obsid_file) == n_frame * 2
		# )
		# or simply
		this_success &= len(this_obsgroup_obsid_file) % n_frame == 0
		if this_instrument == "HSTDM":
		# 不确定以后是1个探测器还是2个探测器
		this_n_file_found = len(this_obsgroup_obsid_file)
		this_n_file_expected = (this_n_frame, this_n_frame * 2)
		this_success &= this_n_file_found in this_n_file_expected
		else:
		# n_detector == n_file
		# this_success &= len(this_obsgroup_obsid_file) == len(
		# effective_detector_names
		# )
		# or more strictly, each detector matches
		this_success &= set(this_obsgroup_obsid_file["detector"]) == set(
		effective_detector_names
		# for other instruments, e.g., MSC
		# n_file_found = len(this_obsgroup_obsid_file)
		# n_file_expected = len(effective_detector_names)
		# this_success &= n_file_found == n_file_expected

		# or more strictly, expected files are a subset of files found
		this_success &= set(this_effective_detector_names) <= set(
		this_obsgroup_obsid_file["detector"]
		)

		# append this task
		@@ -504,8 +539,10 @@ class Dispatcher:
		dict(
		task=this_task,
		success=this_success,
		relevant_plan=this_obsgroup_obsid,
		relevant_plan=this_obsgroup_plan,
		relevant_data=this_obsgroup_file,
		n_relevant_plan=len(this_obsgroup_plan),
		n_relevant_data=this_obsgroup_file,
		)
		)
		return task_list
		@@ -522,6 +559,7 @@ class Dispatcher:
		_["n_frame"] = (
		_["params"]["n_epec_frame"] if _["instrument"] == "HSTDM" else 1
		)
		# 未来如果HSTDM的设定简化一些，这里n_frame可以改成n_file，更直观
		plan_basis = extract_basis_table(
		plan_recs.data,
		PLAN_BASIS_KEYS,
		@@ -531,24 +569,3 @@ class Dispatcher:
		DATA_BASIS_KEYS,
		)
		return plan_basis, data_basis


		# # 1221 plan recs, 36630 data recs
		# plan_basis, data_basis = Dispatcher.load_test_data()
		#
		# # 430 task/s
		# task_list_via_file = Dispatcher.dispatch_file(plan_basis, data_basis)
		#
		# # 13 task/s @n_jobs=1, 100*10 task/s @n_jobs=10 (max)
		# task_list_via_detector = Dispatcher.dispatch_detector(plan_basis, data_basis, n_jobs=10)
		#
		# # 16 task/s @n_jobs=1, 130*10 tasks/s @n_jobs=10 (max) 🔼
		# task_list_via_obsid = Dispatcher.dispatch_obsid(plan_basis, data_basis, n_jobs=10)
		#
		# # 13s/task
		# task_list_via_obsgroup = Dispatcher.dispatch_obsgroup(plan_basis, data_basis)
		# print(
		# sum(_["success"] for _ in task_list_via_obsgroup),
		# "/",
		# len(task_list_via_obsgroup),
		# )