refactor: n_frame -> n_file (8c272e9d) · Commits · csst-cicd / csst-dag

csst_dag/_csst/observation.py

+2 −2

Original line number	Diff line number	Diff line
		@@ -44,8 +44,8 @@ class CsstPlanObsid(DotDict):
		def expected_data_table(self):
		contents = []
		for detector in self.detectors:
		n_frame = self.params.num_epec_frame if self.instrument == "HSTDM" else 1
		for _ in range(n_frame):
		n_file = self.params.num_epec_frame if self.instrument == "HSTDM" else 1
		for _ in range(n_file):
		contents.append(
		dict(
		dataset=self.dataset,

csst_dag/dag/_dispatcher.py

+37 −17

Original line number	Diff line number	Diff line
		@@ -59,7 +59,7 @@ PLAN_BASIS_KEYS = (
		"obs_type",
		"obs_group",
		"obs_id",
		"n_frame",
		"n_file",
		"_id",
		)

		@@ -162,9 +162,14 @@ class Dispatcher:
		# plan basis / obsid basis
		try:
		for _ in qr.data:
		_["n_frame"] = (
		_["params"]["num_epec_frame"] if _["instrument"] == "HSTDM" else 1
		)
		if _["instrument"] == "HSTDM":
		if _["params"]["detector"] == "SIS12":
		this_n_file = _["params"]["num_epec_frame"] * 2
		else:
		this_n_file = _["params"]["num_epec_frame"]
		else:
		this_n_file = 1
		_["n_file"] = this_n_file
		except KeyError:
		print(f"`n_epec_frame` is not found in {_}")
		raise KeyError(f"`n_epec_frame` is not found in {_}")
		@@ -384,7 +389,7 @@ class Dispatcher:
		)

		n_file_expected = (
		this_data_detector_plan["n_frame"][0]
		this_data_detector_plan["n_file"][0]
		if len(this_data_detector_plan) > 0
		else 0
		)
		@@ -411,7 +416,7 @@ class Dispatcher:
		if len(this_data_detector_files) == 0
		else list(this_data_detector_files["_id_data"])
		),
		n_file_expected=this_data_detector_plan["n_frame"].sum(),
		n_file_expected=this_data_detector_plan["n_file"].sum(),
		n_file_found=len(this_data_detector_files),
		)
		)
		@@ -488,8 +493,8 @@ class Dispatcher:

		# whether effective detectors all there
		this_instrument = this_data_obsid["instrument"][0]
		this_n_frame = (
		this_data_obsid_plan["n_frame"] if len(this_data_obsid_plan) > 0 else 0
		this_n_file = (
		this_data_obsid_plan["n_file"] if len(this_data_obsid_plan) > 0 else 0
		)
		this_effective_detector_names = csst[
		this_instrument
		@@ -498,7 +503,7 @@ class Dispatcher:
		if this_instrument == "HSTDM":
		# 不确定以后是1个探测器还是2个探测器
		this_n_file_found = len(this_data_obsid_file)
		this_n_file_expected = (this_n_frame, this_n_frame * 2)
		this_n_file_expected = (this_n_file, this_n_file * 2)
		this_success = this_n_file_found in this_n_file_expected
		else:
		# for other instruments, e.g., MSC
		@@ -511,7 +516,7 @@ class Dispatcher:
		this_data_obsid_file["detector"]
		)

		n_file_expected = int(this_data_obsid_plan["n_frame"].sum())
		n_file_expected = int(this_data_obsid_plan["n_file"].sum())
		n_file_found = len(this_data_obsid_file)
		# set n_file_expected and n_file_found
		this_task["n_file_expected"] = n_file_expected
		@@ -530,13 +535,29 @@ class Dispatcher:
		if len(this_data_obsid_file) == 0
		else list(this_data_obsid_file["_id_data"])
		),
		n_file_expected=this_data_obsid_plan["n_frame"].sum(),
		n_file_expected=this_data_obsid_plan["n_file"].sum(),
		n_file_found=len(this_data_obsid_file),
		)
		)

		return task_list

		@staticmethod
		def disptach_obsgroup_detector(
		plan_basis: table.Table,
		data_basis: table.Table,
		n_jobs: int = 1,
		):
		# unique obsgroup basis
		obsgroup_basis = table.unique(
		plan_basis[
		"dataset",
		"instrument",
		"obs_type",
		"obs_group",
		]
		)

		@staticmethod
		def dispatch_obsgroup(
		plan_basis: table.Table,
		@@ -583,7 +604,7 @@ class Dispatcher:
		# i_obsid = 1
		# print(i_obsid)
		this_instrument = this_obsgroup_plan[i_obsid]["instrument"]
		this_n_frame = this_obsgroup_plan[i_obsid]["n_frame"]
		this_n_file = this_obsgroup_plan[i_obsid]["n_file"]
		this_effective_detector_names = csst[
		this_instrument
		].effective_detector_names
		@@ -599,7 +620,7 @@ class Dispatcher:
		if this_instrument == "HSTDM":
		# 不确定以后是1个探测器还是2个探测器
		this_n_file_found = len(this_obsgroup_obsid_file)
		this_n_file_expected = (this_n_frame, this_n_frame * 2)
		this_n_file_expected = (this_n_file, this_n_file * 2)
		this_success &= this_n_file_found in this_n_file_expected
		else:
		# for other instruments, e.g., MSC
		@@ -612,7 +633,7 @@ class Dispatcher:
		this_obsgroup_obsid_file["detector"]
		)

		n_file_expected = int(this_obsgroup_plan["n_frame"].sum())
		n_file_expected = int(this_obsgroup_plan["n_file"].sum())
		n_file_found = len(this_obsgroup_file)
		# set n_file_expected and n_file_found
		this_task["n_file_expected"] = n_file_expected
		@@ -631,7 +652,7 @@ class Dispatcher:
		if len(this_obsgroup_file) == 0
		else list(this_obsgroup_file["_id_data"])
		),
		n_file_expected=this_obsgroup_plan["n_frame"].sum(),
		n_file_expected=this_obsgroup_plan["n_file"].sum(),
		n_file_found=len(this_obsgroup_file),
		)
		)
		@@ -649,10 +670,9 @@ class Dispatcher:
		print(f"{len(plan_recs.data)} plan records")
		print(f"{len(data_recs.data)} data records")
		for _ in plan_recs.data:
		_["n_frame"] = (
		_["n_file"] = (
		_["params"]["num_epec_frame"] if _["instrument"] == "HSTDM" else 1
		)
		# 未来如果HSTDM的设定简化一些，这里n_frame可以改成n_file，更直观
		plan_basis = extract_basis_table(
		plan_recs.data,
		PLAN_BASIS_KEYS,

tests/test_dispatcher.py

+2 −2

Original line number	Diff line number	Diff line
		@@ -50,6 +50,6 @@ print(task_list_via_obsgroup[0]["relevant_data"].colnames)


		# relevant plan_basis:
		# ['dataset', 'instrument', 'obs_type', 'obs_group', 'obs_id', 'detector', 'n_frame', '_id']
		# ['dataset', 'instrument', 'obs_type', 'obs_group', 'obs_id', 'detector', 'n_file', '_id']
		# relevant data_basis:
		# ['dataset', 'instrument', 'obs_type', 'obs_group', 'obs_id', 'detector', 'file_name', '_id']
		# ['dataset', 'instrument', 'obs_type', 'obs_group', 'obs_id', 'detector', 'file_name', '_id', 'prc_status']