Commit 128f1919 authored by BO ZHANG's avatar BO ZHANG 🏀
Browse files

added clear_dir option and updated get_aux methods

parent e778f976

csst_common/data_manager.py

+27 −22
Original line number Diff line number Diff line
@@ -136,6 +136,7 @@ class CsstMsDataManager: 
                 _l0_post: str = "L0_1",

                 log_ppl="csst-l1ppl.log",

                 log_mod="csst-l1mod.log",

                 clear_dir=False,

                 ):



        # version
@@ -200,6 +201,9 @@ class CsstMsDataManager: 


        # change to working directory

        os.chdir(self.dir_l1)

        # clear dir_l1

        if clear_dir:

            self.clear_dir(self.dir_l1)



        # pipeline logger

        self.logger_ppl = get_logger(name="CSST L1 Pipeline Logger", filename=os.path.join(dir_l1, log_ppl))
@@ -236,7 +240,7 @@ class CsstMsDataManager: 
        raise DeprecationWarning("This method is deprecated, please directly use dm.target_detectors = detectors!")



    @staticmethod

    def from_dfs(ver_sim="C5.2", datatype="mbi", exp_id=10000100, dir_l1="."):

    def from_dfs(expid, ver_sim="C5.2", datatype="mbi", dir_l1="."):

        raise NotImplementedError("from_dfs is currently not available!")



    @staticmethod
@@ -364,24 +368,21 @@ class CsstMsDataManager: 
    def get_bias(self, detector=6):

        """ get bias data """

        if self.datatype == "mbi":

            fp = glob.glob(self.path_aux.format("CLB", detector))[0]

            return fits.getdata(fp)

            return glob.glob(self.path_aux.format("CLB", detector))[0]

        else:

            return self.path_aux.format(detector, "bias")



    def get_dark(self, detector=6):

        """ get dark data """

        if self.datatype == "mbi":

            fp = glob.glob(self.path_aux.format("CLD", detector))[0]

            return fits.getdata(fp)

            return glob.glob(self.path_aux.format("CLD", detector))[0]

        else:

            return self.path_aux.format(detector, "dark")



    def get_flat(self, detector=6):

        """ get flat data """

        if self.datatype == "mbi":

            fp = glob.glob(self.path_aux.format("CLF", detector))[0]

            return fits.getdata(fp)

            return glob.glob(self.path_aux.format("CLF", detector))[0]

        else:

            return self.path_aux.format(detector, "flat")
@@ -430,8 +431,10 @@ class CsstMsDataManager: 
        return info



    @staticmethod

    def quickstart(ver_sim="C5.2", datatype="mbi", dir_l1=".", exposure_id=100, dfs_mode=False, dfs_node="kmust"):

        """Quick dataset generator for tests on dandelion or PMO

    def quickstart(ver_sim="C5.2", datatype="mbi", dir_l1=".", exposure_id=100,

                   dfs_mode=False, dfs_node="kmust", clear_dir=False):

        """

        Quick dataset generator for tests on dandelion or PML



        Parameters

        ----------
@@ -447,6 +450,8 @@ class CsstMsDataManager: 
            If True, use DFS.

        dfs_node : str

            The DFS node. Defaults to "kmust", could be "pml".

        clear_dir : bool

            If True, clear dir_l1.



        Returns

        -------
@@ -487,15 +492,15 @@ class CsstMsDataManager: 
    def __repr__(self):

        lines = ""

        lines += "<CsstMsDataManager>\n"

        lines += f"Data type = {self.datatype}\n"

        lines += f"Valid detectors = {self.valid_detectors}\n"

        lines += f"Available detectors = {self.available_detectors}\n"

        lines += f"Target detectors = {self.target_detectors}\n"

        lines += f"dir_l0 = {self.dir_l0}\n"

        lines += f"dir_l1 = {self.dir_l1}\n"

        lines += f"dfs_mode = {self.dfs_mode}\n"

        lines += f"dfs_node = {self.dfs_node}\n"

        lines += f"CSST_DFS_GATEWAY = " + os.getenv("CSST_DFS_GATEWAY") + "\n"

        lines += f"- Data type = {self.datatype}\n"

        lines += f"- Valid detectors = {self.valid_detectors}\n"

        lines += f"- Available detectors = {self.available_detectors}\n"

        lines += f"- Target detectors = {self.target_detectors}\n"

        lines += f"- dir_l0 = {self.dir_l0}\n"

        lines += f"- dir_l1 = {self.dir_l1}\n"

        lines += f"- dfs_mode = {self.dfs_mode}\n"

        lines += f"- dfs_node = {self.dfs_node}\n"

        lines += f"- CSST_DFS_GATEWAY = " + os.getenv("CSST_DFS_GATEWAY") + "\n"

        return lines



    def remove_files(self, fmt="*.fits"):
@@ -523,8 +528,8 @@ class CsstMsDataManager: 
        for k, v in DFS_CONF[dfs_node].items():

            os.environ[k] = v



    @property

    def dfs_is_available(self):

    @staticmethod

    def dfs_is_available():

        """ Test if DFS works. """

        try:

            tbl = gaia_query_from_dfs(radius=0.001)
@@ -553,10 +558,10 @@ class CsstMsDataManager: 
        """ Push SLS spectra to DFS. """

        pass



    def dfs_l0_query(self):

    def dfs_l0_query(self, obs_id):

        """ Query L0 data from DFS. """

        pass



    def dfs_l1_query(self):

    def dfs_l1_query(self, obs_id, detector):

        """ Query L1 data from DFS. """

        pass