c3

.gitignore

@@ -39,3 +39,4 @@ test_reports.xml
 **/.eggs
 
 **/_version.py
+examples/test.py

csst_dfs_api/common/constants.py

 MODE_LOCAL = 'local'
-MODE_CLUSTER = 'cluster'
\ No newline at end of file
+MODE_CLUSTER = 'cluster'
+
+
+MODULE_MSC = "MSC"
+MODULE_IFS = "IFS"
+MODULE_MCI = "MCI"
+MODULE_CPIC = "CPIC"
+MODULE_HSTDM = "HSTDM"
+
+MODULE_IDS = [MODULE_MSC, MODULE_IFS, MODULE_MCI, MODULE_CPIC, MODULE_HSTDM]

csst_dfs_api/facility/__init__.py

+from .calmerge import CalMergeApi
+from .detector import DetectorApi
+from .level0 import Level0DataApi
+from .level0prc import Level0PrcApi
+from .observation import ObservationApi
\ No newline at end of file

csst_dfs_api/facility/calmerge.py

+
+from ..common.delegate import Delegate
+
+class CalMergeApi(object):
+    """
+    Level 0 Data Operation API
+    """
+    def __init__(self):
+        self.pymodule = Delegate().load(sub_module = "facility")
+        self.stub = getattr(self.pymodule, "CalMergeApi")()
+        
+    def find(self, **kwargs):
+        ''' retrieve calibration merge records from database
+
+        parameter kwargs:
+            detector_no: [str]
+            ref_type: [str]
+            obs_time: (start,end)
+            qc1_status : [int]
+            prc_status : [int]
+            file_name: [str]
+            limit: limits returns the number of records,default 0:no-limit
+
+        return: csst_dfs_common.models.Result
+        '''
+        return self.stub.find(**kwargs)
+
+    def get(self, **kwargs):
+        '''  fetch a record from database
+
+        parameter kwargs:
+            id : [int] 
+
+        return csst_dfs_common.models.Result
+        '''
+        return self.stub.get(**kwargs)
+
+    def update_proc_status(self, **kwargs):
+        ''' update the status of reduction
+
+        parameter kwargs:
+            id : [int],
+            status : [int]
+
+        return csst_dfs_common.models.Result
+        '''
+        return self.stub.update_proc_status(**kwargs)
+
+    def update_qc1_status(self, **kwargs):
+        ''' update the status of reduction
+
+        parameter kwargs:
+            id : [int],
+            status : [int]
+
+        return csst_dfs_common.models.Result
+        '''        
+        return self.stub.update_qc1_status(**kwargs)    
+
+    def write(self, **kwargs):
+        ''' insert a calibration merge record into database
+ 
+        parameter kwargs:
+            id : [int]
+            detector_no : [str]
+            ref_type : [str]
+            obs_time : [str]
+            exp_time : [float]
+            prc_status : [int]
+            prc_time : [str]
+            filename : [str]
+            file_path : [str]
+            level0_ids : [list]
+        return csst_dfs_common.models.Result
+        '''  
+        return self.stub.write(**kwargs)
+

csst_dfs_api/facility/detector.py

+from ..common.delegate import Delegate
+
+class DetectorApi(object):
+    """
+    Detector Operation Class
+    """
+    def __init__(self):
+        self.pymodule = Delegate().load(sub_module = "facility")
+        self.stub = getattr(self.pymodule, "DetectorApi")()
+        
+    def find(self, **kwargs):
+        ''' retrieve detector records from database
+
+        parameter kwargs:
+            module_id: [str]
+            key: [str]
+
+        return: csst_dfs_common.models.Result
+        '''
+        return self.stub.find(**kwargs)
+
+    def get(self, **kwargs):
+        '''  fetch a record from database
+
+        parameter kwargs:
+            no : [str] 
+
+        return csst_dfs_common.models.Result
+        '''
+        return self.stub.get(**kwargs)
+
+    def update(self, **kwargs):
+        ''' update a detector by no
+
+        parameter kwargs:
+            no : [str],
+            detector_name : [str],
+            module_id : [str],
+            filter_id : [str]
+
+        return csst_dfs_common.models.Result
+        '''
+        return self.stub.update(**kwargs)
+
+    def delete(self, **kwargs):
+        ''' delete a detector by no
+
+        parameter kwargs:
+            no : [str]
+
+        return csst_dfs_common.models.Result
+        '''
+        return self.stub.delete(**kwargs)
+
+    def write(self, **kwargs):
+        ''' insert a detector record into database
+ 
+        parameter kwargs:
+            no : [str],
+            detector_name : [str],
+            module_id : [str],
+            filter_id : [str]
+        return csst_dfs_common.models.Result
+        '''        
+        return self.stub.write(**kwargs)
+
+    def find_status(self, **kwargs):
+        ''' retrieve a detector status's from database
+
+        parameter kwargs:
+            detector_no: [str]
+            status_occur_time: (begin,end)
+            limit: limits returns the number of records,default 0:no-limit
+
+        return: csst_dfs_common.models.Result
+        '''
+        return self.stub.find_status(**kwargs)
+
+    def get_status(self, **kwargs):
+        '''  fetch a record from database
+
+        parameter kwargs:
+            id : [int] 
+
+        return csst_dfs_common.models.Result
+        '''
+        return self.stub.get_status(**kwargs)
+
+    def write_status(self, **kwargs):
+        ''' insert a detector status into database
+ 
+        parameter kwargs:
+            detector_no : [str],
+            status : [str],
+            status_time : [str]
+        return csst_dfs_common.models.Result
+        '''
+        return self.stub.write_status(**kwargs)  
\ No newline at end of file

csst_dfs_api/facility/level0.py

+
+from ..common.delegate import Delegate
+
+class Level0DataApi(object):
+    """
+    Level 0 Data Operation API
+    """
+    def __init__(self):
+        self.pymodule = Delegate().load(sub_module = "facility")
+        self.stub = getattr(self.pymodule, "Level0DataApi")()
+        
+    def find(self, **kwargs):
+        ''' retrieve level0 records from database
+
+        parameter kwargs:
+            obs_id: [int]
+            detector_no: [str]
+            obs_type: [str]
+            obs_time : (start, end),
+            qc0_status : [int],
+            prc_status : [int],
+            file_name: [str]
+            limit: limits returns the number of records,default 0:no-limit
+
+        return: csst_dfs_common.models.Result
+        '''
+        return self.stub.find(**kwargs)
+
+    def get(self, **kwargs):
+        '''  fetch a record from database
+
+        parameter kwargs:
+            fits_id : [int] 
+
+        return csst_dfs_common.models.Result
+        '''
+        return self.stub.get(**kwargs)
+
+    def update_proc_status(self, **kwargs):
+        ''' update the status of reduction
+
+        parameter kwargs:
+            fits_id : [int],
+            status : [int]
+        '''
+        return self.stub.update_proc_status(**kwargs)
+
+    def update_qc0_status(self, **kwargs):
+        ''' update the status of QC0
+        
+        parameter kwargs:
+            fits_id : [int],
+            status : [int]
+        '''        
+        return self.stub.update_qc0_status(**kwargs)    
+
+    def write(self, **kwargs):
+        ''' insert a level0 data record into database
+ 
+        parameter kwargs:
+            obs_id = [int]
+            detector_no = [str]
+            obs_type = [str]        
+            obs_time = [str]
+            exp_time = [int]
+            detector_status_id = [int]
+            filename = [str]
+            file_path = [str]
+        return: csst_dfs_common.models.Result
+        '''  
+        return self.stub.write(**kwargs)
+

csst_dfs_api/facility/level0prc.py

+
+from ..common.delegate import Delegate
+
+class Level0PrcApi(object):
+    """
+    Level 0 Data Operation API
+    """
+    def __init__(self):
+        self.pymodule = Delegate().load(sub_module = "facility")
+        self.stub = getattr(self.pymodule, "Level0PrcApi")()
+        
+    def find(self, **kwargs):
+        ''' retrieve level0 procedure records from database
+
+        parameter kwargs:
+            level0_id: [int]
+            pipeline_id: [str]
+            prc_module: [str]
+            prc_status : [int]
+
+        return: csst_dfs_common.models.Result
+        '''
+        return self.stub.find(**kwargs)
+
+    def update_proc_status(self, **kwargs):
+        ''' update the status of reduction
+
+        parameter kwargs:
+            id : [int],
+            status : [int]
+
+        return csst_dfs_common.models.Result
+        '''
+        return self.stub.update_proc_status(**kwargs)
+
+    def write(self, **kwargs):
+        ''' insert a level0 procedure record into database
+ 
+        parameter kwargs:
+            level0_id : [int]
+            pipeline_id : [str]
+            prc_module : [str]
+            params_id : [str]
+            prc_status : [int]
+            prc_time : [str]
+            file_path : [str]
+        return csst_dfs_common.models.Result
+        '''  
+        return self.stub.write(**kwargs)
+

csst_dfs_api/ifs/fits.py → csst_dfs_api/facility/observation.py

 
 from ..common.delegate import Delegate
 
-class FitsApi(object):
+class ObservationApi(object):
     """
-    Raw Fits Operation Class of IFS
+    Observation Operation Class
     """
-    def __init__(self, sub_system="ifs"):
-        self.sub_system = sub_system
-        self.module = Delegate().load(sub_module = "ifs")
-        self.stub = getattr(self.module, "FitsApi")()
+    def __init__(self):
+        self.pymodule = Delegate().load(sub_module = "facility")
+        self.stub = getattr(self.pymodule, "ObservationApi")()
         
     def find(self, **kwargs):
-        '''
+        ''' retrieve exposure records from database
+
         parameter kwargs:
-            obs_time = [int],
-            file_name = [str],
-            exp_time = (start, end),
-            ccd_num = [int],
-            qc0_status = [int],
-            prc_status = [int],
-            limit: limits returns the number of records
+            module_id: [str]
+            obs_type: [str]
+            obs_time : (start, end),
+            qc0_status : [int],
+            prc_status : [int],
+            limit: limits returns the number of records,default 0:no-limit
 
         return: csst_dfs_common.models.Result
         '''
         return self.stub.find(**kwargs)
 
     def get(self, **kwargs):
-        '''  query database, return a record as dict
+        '''  fetch a record from database
 
         parameter kwargs:
-            fits_id = [int] 
+            obs_id = [int] 
 
-        return dict or None
+        return: csst_dfs_common.models.Result
         '''
         return self.stub.get(**kwargs)
 
-    def read(self, **kwargs):
-        ''' yield bytes of fits file
-
-        parameter kwargs:
-            fits_id = [int],
-            file_path = [str], 
-            chunk_size = [int] default 20480
-
-        yield bytes of fits file
-        '''
-        return self.stub.read(**kwargs)
 
     def update_proc_status(self, **kwargs):
         ''' update the status of reduction
 
         parameter kwargs:
-            fits_id = [int],
+            obs_id = [int],
             status = [int]
+        
+        return: csst_dfs_common.models.Result
         '''
         return self.stub.update_proc_status(**kwargs)
 
     def update_qc0_status(self, **kwargs):
-        ''' update the status of reduction
+        ''' update the status of QC0
         
         parameter kwargs:
-            fits_id = [int],
+            obs_id = [int],
             status = [int]
+
+        return: csst_dfs_common.models.Result
         '''        
         return self.stub.update_qc0_status(**kwargs)    
 
     def write(self, **kwargs):
-        ''' copy a local file to file storage, then reduce the header of fits file and insert a record into database
+        ''' insert a observational record into database
  
         parameter kwargs:
-            file_path = [str]
+            obs_time = [str]
+            exp_time = [float]
+            module_id = [str]
+            obs_type = [str]
+            facility_status_id = [int]
+            module_status_id = [int]
+        
+        return: csst_dfs_common.models.Result
         '''        
         return self.stub.write(**kwargs)
 

csst_dfs_api/ifs/__init__.py

-from .fits import FitsApi
-from .reffits import RefFitsApi
-from .result0 import Result0Api
-from .result1 import Result1Api
\ No newline at end of file
+from .level1 import Level1DataApi
\ No newline at end of file

csst_dfs_api/ifs/level1.py

+
+from ..common.delegate import Delegate
+
+class Level1DataApi(object):
+    """
+    Level1 Data Operation Class
+    """
+    def __init__(self):
+        self.pymodule = Delegate().load(sub_module = "ifs")
+        self.stub = getattr(self.pymodule, "Level1DataApi")()
+        
+    def find(self, **kwargs):
+        ''' retrieve level1 records from database
+
+        parameter kwargs:
+            raw_id: [int]
+            data_type: [str]
+            obs_type: [str]
+            create_time : (start, end),
+            qc1_status : [int],
+            prc_status : [int],
+            filename: [str]
+            limit: limits returns the number of records,default 0:no-limit
+
+        return: csst_dfs_common.models.Result
+        '''
+        return self.stub.find(**kwargs)
+
+    def get(self, **kwargs):
+        '''  fetch a record from database
+
+        parameter kwargs:
+            id : [int] 
+
+        return csst_dfs_common.models.Result
+        '''
+        return self.stub.get(**kwargs)
+
+
+    def update_proc_status(self, **kwargs):
+        ''' update the status of reduction
+
+        parameter kwargs:
+            id = [int],
+            status = [int]
+        
+        return: csst_dfs_common.models.Result
+        '''
+        return self.stub.update_proc_status(**kwargs)
+
+    def update_qc1_status(self, **kwargs):
+        ''' update the status of QC0
+        
+        parameter kwargs:
+            id = [int],
+            status = [int]
+
+        return: csst_dfs_common.models.Result
+        '''        
+        return self.stub.update_qc1_status(**kwargs)    
+
+    def write(self, **kwargs):
+        ''' insert a level1 record into database
+ 
+        parameter kwargs:
+            raw_id : [int]
+            data_type : [str]
+            cor_sci_id : [int]
+            prc_params : [str]
+            flat_id : [int]
+            dark_id : [int]
+            bias_id : [int]
+            lamp_id : [int]
+            arc_id : [int]
+            sky_id : [int]            
+            filename : [str]
+            file_path : [str]            
+            prc_status : [int]
+            prc_time : [str]
+            pipeline_id : [str]
+
+        return csst_dfs_common.models.Result
+        '''          
+        return self.stub.write(**kwargs)
+

csst_dfs_api/ifs/reffits.py

-
-from ..common.delegate import Delegate
-
-class RefFitsApi(object):
-    """
-    Reference Fits Operation Class of IFS
-    """
-    REF_FITS_BIAS = "bias"
-    REF_FITS_FLAT = "flat"
-    REF_FITS_DARK = "dark"
-    REF_FITS_SKY = "sky"
-    REF_FITS_ARC = "arc"
-
-    REF_FITS_TYPES = [REF_FITS_BIAS, REF_FITS_FLAT, REF_FITS_DARK, REF_FITS_SKY, REF_FITS_ARC]
-
-    def __init__(self, sub_system="ifs"):
-        self.sub_system = sub_system
-        self.module = Delegate().load(sub_module = "ifs")
-        self.stub = getattr(self.module, "RefFitsApi")()
-        self.file_prefix = self.stub.root_dir
-
-    def find(self, **kwargs):
-        '''
-        parameter kwargs:
-            obs_time = [int],
-            file_name = [str],
-            exp_time = (start, end),
-            ccd_num = [int],
-            status = [int],
-            ref_type = [str]
-
-        return list of reference's files records
-        '''
-        return self.stub.find(**kwargs)
-
-    def get(self, **kwargs):
-        '''  query database, return a record as dict
-
-        parameter kwargs:
-            fits_id = [int] 
-
-        return dict or None
-        '''
-        return self.stub.get(**kwargs)
-
-    def read(self, **kwargs):
-        ''' yield bytes of fits file
-
-        parameter kwargs:
-            fits_id = [int],
-            file_path = [str], 
-            chunk_size = [int] default 20480
-
-        yield bytes of fits file
-        '''
-        return self.stub.read(**kwargs)
-
-    def update_status(self, **kwargs):
-        ''' update the status of reduction
-
-        parameter kwargs:
-            fits_id = [int],
-            status = [int]
-        '''
-        return self.stub.update_status(**kwargs)
-
-    def write(self, **kwargs):
-        ''' copy a local file to file storage, then reduce the header of fits file and insert a record into database
- 
-        parameter kwargs:
-            file_path = [str]
-        '''        
-        return self.stub.write(**kwargs)
-

csst_dfs_api/ifs/result0.py

-
-from ..common.delegate import Delegate
-
-class Result0Api(object):
-    """
-    Level 0 Operation Class of IFS
-    """
-    def __init__(self, sub_system="ifs"):
-        self.sub_system = sub_system
-        self.module = Delegate().load(sub_module = "ifs")
-        self.stub = getattr(self.module, "Result0Api")()
-        self.file_prefix = self.stub.root_dir
-
-    def find(self, **kwargs):
-        '''
-        parameter kwargs:
-            raw_id = [int],
-            file_name = [str],
-            proc_type = [str]
-
-        return list of level 0 record
-        '''
-        return self.stub.find(**kwargs)
-
-    def get(self, **kwargs):
-        '''  query database, return a record as dict
-
-        parameter kwargs:
-            fits_id = [int] 
-
-        return dict or None
-        '''
-        return self.stub.get(**kwargs)
-
-    def read(self, **kwargs):
-        ''' yield bytes of fits file
-
-        parameter kwargs:
-            fits_id = [int],
-            file_path = [str], 
-            chunk_size = [int] default 20480
-
-        yield bytes of fits file
-        '''
-        return self.stub.read(**kwargs)
-
-    def write(self, **kwargs):
-        ''' copy a local level 0 file to file storage, and insert a record into database
-
-        parameter kwargs:
-            raw_id = [int],
-            file_path = [str],
-            proc_type = [str]
-        '''  
-        return self.stub.write(**kwargs)
-

csst_dfs_api/ifs/result1.py

-
-from ..common.delegate import Delegate
-
-class Result1Api(object):
-    """
-    Level 1 Operation Class of IFS
-    """
-    def __init__(self, sub_system="ifs"):
-        self.sub_system = sub_system
-        self.module = Delegate().load(sub_module = "ifs")
-        self.stub = getattr(self.module, "Result1Api")()
-        self.file_prefix = self.stub.root_dir
-
-    def find(self, **kwargs):
-        '''
-        parameter kwargs:
-            file_name = [str],
-            proc_type = [str]
-
-        return list of level 1 record
-        '''
-        return self.stub.find(**kwargs)
-
-    def get(self, **kwargs):
-        '''  query database, return a record as dict
-
-        parameter kwargs:
-            fits_id = [int] 
-
-        return dict or None
-        '''
-        return self.stub.get(**kwargs)
-
-    def read(self, **kwargs):
-        ''' yield bytes of fits file
-
-        parameter kwargs:
-            fits_id = [int],
-            file_path = [str], 
-            chunk_size = [int] default 20480
-
-        yield bytes of fits file
-        '''
-        return self.stub.read(**kwargs)
-
-    def write(self, **kwargs):
-        ''' copy a local level 1 file to file storage, and insert a record into database
-
-        parameter kwargs:
-            file_path = [str],
-            proc_type = [str],
-            result0_ids = [list]
-        '''  
-        return self.stub.write(**kwargs)
-

csst_dfs_api/msc/__init__.py

+from .level1 import Level1DataApi
\ No newline at end of file

csst_dfs_api/msc/level1.py

+
+from ..common.delegate import Delegate
+
+class Level1DataApi(object):
+    """
+    Level1 Data Operation Class
+    """
+    def __init__(self):
+        self.pymodule = Delegate().load(sub_module = "msc")
+        self.stub = getattr(self.pymodule, "Level1DataApi")()
+        
+    def find(self, **kwargs):
+        ''' retrieve level1 records from database
+
+        parameter kwargs:
+            raw_id: [int]
+            data_type: [str]
+            obs_type: [str]
+            create_time : (start, end),
+            qc1_status : [int],
+            prc_status : [int],
+            filename: [str]
+            limit: limits returns the number of records,default 0:no-limit
+
+        return: csst_dfs_common.models.Result
+        '''
+        return self.stub.find(**kwargs)
+
+    def get(self, **kwargs):
+        '''  fetch a record from database
+
+        parameter kwargs:
+            id : [int] 
+
+        return csst_dfs_common.models.Result
+        '''
+        return self.stub.get(**kwargs)
+
+
+    def update_proc_status(self, **kwargs):
+        ''' update the status of reduction
+
+        parameter kwargs:
+            id = [int],
+            status = [int]
+        
+        return: csst_dfs_common.models.Result
+        '''
+        return self.stub.update_proc_status(**kwargs)
+
+    def update_qc1_status(self, **kwargs):
+        ''' update the status of QC0
+        
+        parameter kwargs:
+            id = [int],
+            status = [int]
+
+        return: csst_dfs_common.models.Result
+        '''        
+        return self.stub.update_qc1_status(**kwargs)    
+
+    def write(self, **kwargs):
+        ''' insert a level1 record into database
+ 
+        parameter kwargs:
+            raw_id : [int]
+            data_type : [str]
+            cor_sci_id : [int]
+            prc_params : [str]
+            flat_id : [int]
+            dark_id : [int]
+            bias_id : [int]
+            filename : [str]
+            file_path : [str]            
+            prc_status : [int]
+            prc_time : [str]
+            pipeline_id : [str]
+
+        return csst_dfs_common.models.Result
+        '''          
+        return self.stub.write(**kwargs)
+

tests/test_common_catalog.py

@@ -9,6 +9,15 @@ class CommonCatalogTestCase(unittest.TestCase):
     def setUp(self):
         self.api = CatalogApi()
 
-    def test_gaia3_query(self):
-        result = self.api.gaia3_query(ra=160, dec=-17, radius=0.2, min_mag=-1, max_mag=-1, obstime = -1, limit = 2)
+    def test_catalog_query(self):
+        result = self.api.catalog_query(
+            ra=160, 
+            dec=-17, 
+            radius=0.2, 
+            catalog_name='gaia3', 
+            min_mag=-1, 
+            max_mag=-1, 
+            obstime = -1, 
+            limit = 2
+        )
         print('return:', result)

tests/test_facility_cal_merge.py

+import os
+import unittest
+from astropy.io import fits
+
+from csst_dfs_api.facility.calmerge import CalMergeApi
+
+class CalMergeApiTestCase(unittest.TestCase):
+
+    def setUp(self):
+        self.api = CalMergeApi()
+
+    def test_find(self):
+        recs = self.api.find(detector_no='CCD01',
+            ref_type = "bias",
+            obs_time = ("2021-06-01 11:12:13","2021-06-08 11:12:13"))
+        print('find:', recs)
+
+    def test_get(self):
+        rec = self.api.get(id = 3)
+        print('get:', rec)
+
+    def test_update_proc_status(self):
+        rec = self.api.update_proc_status(id = 3, status = 1)
+        print('update_proc_status:', rec)
+
+    def test_update_qc1_status(self):
+        rec = self.api.update_qc1_status(id = 3, status = 2)
+        print('update_qc1_status:', rec)    
+
+    def test_write(self):
+        rec = self.api.write(detector_no='CCD01', 
+            ref_type = "bias",
+            obs_time = "2021-06-04 11:12:13",
+            exp_time = 150,
+            filename = "/opt/dddasd.params",
+            file_path = "/opt/dddasd.fits",
+            prc_status = 3,
+            prc_time = '2021-06-04 11:12:13',
+            level0_ids = [1,2,3,4])
+        print('write:', rec)
\ No newline at end of file