Commit 442fb3c7 authored by BO ZHANG's avatar BO ZHANG 🏀
Browse files

update dm.dfs_is_available()

parent 51c6e403

csst_common/data_manager.py

+14 −17
Original line number Diff line number Diff line
@@ -26,8 +26,8 @@ from csst_dfs_api.msc.level0 import Level0DataApi 
from csst_dfs_api.msc.level0prc import Level0PrcApi

from csst_dfs_api.msc.level1 import Level1DataApi

from csst_dfs_api.msc.level2 import Level2DataApi

from csst_dfs_api.common.catalog import CatalogApi



from .dfs import gaia_query_from_dfs

from .logger import get_logger

from .params import CSST_PARAMS as CP

from .params import DFS_CONF
@@ -248,6 +248,10 @@ class CsstMsDataManager: 
    def dfs_CalApi(self):

        return CalMergeApi()



    @property

    def dfs_CatApi(self):

        return CatalogApi()



    def set_env(self):

        """ set environment variables """

        if os.uname()[1] == "dandelion":
@@ -567,17 +571,21 @@ class CsstMsDataManager: 
    def dfs_is_available():

        """ Test if DFS works. """

        try:

            tbl = gaia_query_from_dfs(radius=0.001)

            assert len(tbl) == 0

            return True

            tbl = CatalogApi().catalog_query(

                catalog_name="gaia3", ra=180, dec=0, radius=.1, min_mag=0, max_mag=30, obstime=-1, limit=-1

            )

            return len(tbl) > 0

        except:

            return False



    def dfs_rc_query(self, ra=180, dec=0, radius=2, min_mag=0, max_mag=30, obstime=-1, limit=-1):

        """ Query Reference Catalog (RC) from DFS. """

        try:

            tbl = gaia_query_from_dfs(

                ra=ra, dec=dec, radius=radius, min_mag=min_mag, max_mag=max_mag, obstime=obstime, limit=limit)

            cat = self.dfs_CatApi.catalog_query(

                catalog_name="gaia3", ra=ra, dec=dec, radius=radius, min_mag=min_mag, max_mag=max_mag,

                obstime=obstime, limit=limit

            )

            tbl = self.dfs_CatApi.to_table(cat)

            return tbl

        except:

            print("Error occurred during the query!")
@@ -686,14 +694,3 @@ class CsstMsDataManager: 


# temporarily compatible with old interface

CsstMbiDataManager = CsstMsDataManager
 




def test_dm():

    import joblib

    from csst_common.data_manager import CsstMsDataManager

    dm = CsstMsDataManager(dfs_node="pml")



    def f(dm, a=1):

        print(a)



    joblib.Parallel(n_jobs=10, )(joblib.delayed(f)(dm) for i in range(10))

csst_common/dfs.py

deleted100644 → 0
+0 −51
Original line number Diff line number Diff line 
import os

from csst_dfs_api.common.catalog import CatalogApi

from csst_dfs_api.msc.level1 import Level1DataApi





# DFS query

def gaia_query_from_dfs(ra=180, dec=0, radius=2, min_mag=0, max_mag=30, obstime=-1, limit=-1):

    """ query gaia edr3 catalog from dfs """

    api = CatalogApi()

    cat = api.catalog_query(

        catalog_name="gaia3", ra=ra, dec=dec, radius=radius, min_mag=min_mag, max_mag=max_mag,

        obstime=obstime, limit=limit

    )

    tbl = api.to_table(cat)

    return tbl





def get_l1api(node="pm"):

    """ get DFS L1 API



    Parameters

    ----------

    node: str

        in present, node has to be "pm"



    Returns

    -------

    DFS L1 API



    Examples

    --------

    >>>

    >>> rec = l1api.write(

    >>>     level0_id='10000000020',

    >>>     data_type = "sci",

    >>>     cor_sci_id = 1,

    >>>     prc_status = 3,

    >>>     prc_time = '2022-01-05 15:18:13',

    >>>     filename = "MSC_MS_210525120000_100000000_20_img.fits",

    >>>     file_path = "/nfs/data/test20211012/output/L1/150s/MSC_MS_210525120000_100000000_20_img.fits",

    >>>     pipeline_id = "P1_test",

    >>>     refs = {})

    >>> print('write:', rec)

    """

    l1api = Level1DataApi()

    return l1api





if __name__ == "__main__":

    tbl = gaia_query_from_dfs()

    print(tbl)