add n_jobs and backend to dm (e66ed791) · Commits · Wu Maochun / csst_common

csst_common/data_manager.py

+45 −22

Original line number	Diff line number	Diff line
		@@ -107,6 +107,10 @@ class CsstMsDataManager:
		If True, clear ``dm.dir_l1`` directory.
		verbose : bool
		If True, print verbose info.
		n_jobs : int
		The number of jobs.
		backend : str
		The joblib backend.

		Examples
		--------
		@@ -152,6 +156,8 @@ class CsstMsDataManager:
		log_mod="csst-l1mod.log",
		clear_dir=False,
		verbose=True,
		n_jobs=18,
		backend="multiprocessing"
		):

		# version
		@@ -211,7 +217,8 @@ class CsstMsDataManager:
		# record hard code names in history
		self.hardcode_history = []

		self.n_jobs = 18
		self.n_jobs = n_jobs
		self.backend = backend

		# aXe
		self.set_env()
		@@ -287,7 +294,17 @@ class CsstMsDataManager:
		raise DeprecationWarning("This method is deprecated, please directly use dm.target_detectors = detectors!")

		@staticmethod
		def from_dir(ver_sim="C5.2", datatype="mbi", dir_l0=".", dir_l1=".", path_aux="", use_dfs=False, dfs_node="kmust"):
		def from_dir(
		ver_sim="C5.2",
		datatype="mbi",
		dir_l0=".",
		dir_l1=".",
		path_aux="",
		use_dfs=False,
		dfs_node="kmust",
		n_jobs=18,
		backend="multiprocessing"
		):
		""" initialize the multi-band imaging data manager """

		assert ver_sim in ["C5.2", ]
		@@ -310,7 +327,8 @@ class CsstMsDataManager:
		# exp_stop = int(exp_stop)
		# obs_id = int(obs_id)

		return CsstMsDataManager(ver_sim=ver_sim,
		return CsstMsDataManager(
		ver_sim=ver_sim,
		datatype=datatype,
		available_detectors=available_detectors,
		target_detectors=None,
		@@ -327,6 +345,8 @@ class CsstMsDataManager:
		_survey=_survey,
		obs_type=obs_type,
		l0_post="_".join(l0_post),
		n_jobs=n_jobs,
		backend=backend
		)

		@staticmethod
		@@ -641,7 +661,9 @@ class CsstMsDataManager:
		use_dfs=True,
		dfs_node="pml",
		clear_l1=False,
		dfs_root="/share/dfs"
		dfs_root="/share/dfs",
		n_jobs=18,
		backend="multiprocessing"
		):
		""" Initialize CsstMsDataManager from DFS. """
		# (clear and) make directories
		@@ -684,7 +706,8 @@ class CsstMsDataManager:
		# initialize dm
		dm = CsstMsDataManager.from_dir(
		ver_sim="C5.2", datatype=datatype, dir_l0=dir_l0, dir_l1=dir_l1,
		path_aux=path_aux, use_dfs=use_dfs, dfs_node=dfs_node
		path_aux=path_aux, use_dfs=use_dfs, dfs_node=dfs_node,
		n_jobs=n_jobs, backend=backend
		)
		assert dm.obs_id == obs_id