add n_jobs and backend to dm (e66ed791) · Commits · csst-pipeline / csst_common

csst_common/data_manager.py

+45 −22

Original line number	Original line	Diff line number	Diff line
	@@ -107,6 +107,10 @@ class CsstMsDataManager:
	If True, clear ``dm.dir_l1`` directory.		If True, clear ``dm.dir_l1`` directory.
	verbose : bool		verbose : bool
	If True, print verbose info.		If True, print verbose info.
			n_jobs : int
			The number of jobs.
			backend : str
			The joblib backend.

	Examples		Examples
	--------		--------
	@@ -152,6 +156,8 @@ class CsstMsDataManager:
	log_mod="csst-l1mod.log",		log_mod="csst-l1mod.log",
	clear_dir=False,		clear_dir=False,
	verbose=True,		verbose=True,
			n_jobs=18,
			backend="multiprocessing"
	):		):

	# version		# version
	@@ -211,7 +217,8 @@ class CsstMsDataManager:
	# record hard code names in history		# record hard code names in history
	self.hardcode_history = []		self.hardcode_history = []

	self.n_jobs = 18		self.n_jobs = n_jobs
			self.backend = backend

	# aXe		# aXe
	self.set_env()		self.set_env()
	@@ -287,7 +294,17 @@ class CsstMsDataManager:
	raise DeprecationWarning("This method is deprecated, please directly use dm.target_detectors = detectors!")		raise DeprecationWarning("This method is deprecated, please directly use dm.target_detectors = detectors!")

	@staticmethod		@staticmethod
	def from_dir(ver_sim="C5.2", datatype="mbi", dir_l0=".", dir_l1=".", path_aux="", use_dfs=False, dfs_node="kmust"):		def from_dir(
			ver_sim="C5.2",
			datatype="mbi",
			dir_l0=".",
			dir_l1=".",
			path_aux="",
			use_dfs=False,
			dfs_node="kmust",
			n_jobs=18,
			backend="multiprocessing"
			):
	""" initialize the multi-band imaging data manager """		""" initialize the multi-band imaging data manager """

	assert ver_sim in ["C5.2", ]		assert ver_sim in ["C5.2", ]
	@@ -310,7 +327,8 @@ class CsstMsDataManager:
	# exp_stop = int(exp_stop)		# exp_stop = int(exp_stop)
	# obs_id = int(obs_id)		# obs_id = int(obs_id)

	return CsstMsDataManager(ver_sim=ver_sim,		return CsstMsDataManager(
			ver_sim=ver_sim,
	datatype=datatype,		datatype=datatype,
	available_detectors=available_detectors,		available_detectors=available_detectors,
	target_detectors=None,		target_detectors=None,
	@@ -327,6 +345,8 @@ class CsstMsDataManager:
	_survey=_survey,		_survey=_survey,
	obs_type=obs_type,		obs_type=obs_type,
	l0_post="_".join(l0_post),		l0_post="_".join(l0_post),
			n_jobs=n_jobs,
			backend=backend
	)		)

	@staticmethod		@staticmethod
	@@ -641,7 +661,9 @@ class CsstMsDataManager:
	use_dfs=True,		use_dfs=True,
	dfs_node="pml",		dfs_node="pml",
	clear_l1=False,		clear_l1=False,
	dfs_root="/share/dfs"		dfs_root="/share/dfs",
			n_jobs=18,
			backend="multiprocessing"
	):		):
	""" Initialize CsstMsDataManager from DFS. """		""" Initialize CsstMsDataManager from DFS. """
	# (clear and) make directories		# (clear and) make directories
	@@ -684,7 +706,8 @@ class CsstMsDataManager:
	# initialize dm		# initialize dm
	dm = CsstMsDataManager.from_dir(		dm = CsstMsDataManager.from_dir(
	ver_sim="C5.2", datatype=datatype, dir_l0=dir_l0, dir_l1=dir_l1,		ver_sim="C5.2", datatype=datatype, dir_l0=dir_l0, dir_l1=dir_l1,
	path_aux=path_aux, use_dfs=use_dfs, dfs_node=dfs_node		path_aux=path_aux, use_dfs=use_dfs, dfs_node=dfs_node,
			n_jobs=n_jobs, backend=backend
	)		)
	assert dm.obs_id == obs_id		assert dm.obs_id == obs_id