reconstruct

README.md

@@ -15,6 +15,7 @@ python setup.py install
 ## Configuration
 
 enviroment variables
-- CSST_DFS_API_MODE = local or cluster         # defaulrt: cluster
+
+- CSST_DFS_API_MODE = local or cluster         # default: local
 - CSST_LOCAL_FILE_ROOT = [a local file directory] # required if DFS_API_MODE = local,  default: /opt/temp/csst
 - CSST_DFS_CONFIG_SERVER = [config server's address] # required if DFS_API_MODE = cluster, 

csst_dfs_api/common/delegate.py

@@ -30,9 +30,18 @@ class Delegate(object):
             if not self.config_server:
                 raise CSSTGenericException("enviroment variable CSST_DFS_CONFIG_SERVER is not set")
             try:
-                from  csst_dfs_api_cluster._version import version as cluster_version
+                from csst_dfs_api_cluster._version import version as cluster_version
             except ImportError:
                 raise CSSTFatalException("please install csst_dfs_api_cluster firstly.")
     
     def load(self, sub_module):
         return importlib.import_module(f"{API_MODULE_PREFIX}{self.mode}.{sub_module}")
+
+    @classmethod
+    def root_dir(self):
+        mode = os.getenv("CSST_DFS_API_MODE",'local')
+        if mode == MODE_LOCAL:
+            return os.getenv("CSST_LOCAL_FILE_ROOT")
+        if mode == MODE_CLUSTER:
+            from csst_dfs_api_cluster.common import config
+            return config.filePrefix

csst_dfs_api/ifs/__init__.py

 from .fits import FitsApi
 from .reffits import RefFitsApi
-from .result0 import Result0Api
\ No newline at end of file
+from .result0 import Result0Api
+from .result1 import Result1Api
\ No newline at end of file

csst_dfs_api/ifs/fits.py

@@ -2,32 +2,74 @@
 from ..common.delegate import Delegate
 
 class FitsApi(object):
+    """
+    Raw Fits Operation Class of IFS
+    """
     def __init__(self, sub_system="ifs"):
         self.sub_system = sub_system
         self.module = Delegate().load(sub_module = "ifs")
         self.stub = getattr(self.module, "FitsApi")()
-
+        self.file_prefix = self.stub.root_dir
+        
     def find(self, **kwargs):
         '''
         parameter kwargs:
-        obs_time = [str] 
+            obs_time = [int],
+            file_name = [str],
+            exp_time = (start, end),
+            ccd_num = [int],
+            qc0_status = [int],
+            prc_status = [int]
 
-        return list of fits_id
+        return list of raw records
         '''
         return self.stub.find(**kwargs)
 
-    def read(self, **kwargs):
+    def get(self, **kwargs):
+        '''  query database, return a record as dict
+
+        parameter kwargs:
+            fits_id = [int] 
+
+        return dict or None
         '''
+        return self.stub.get(**kwargs)
+
+    def read(self, **kwargs):
+        ''' yield bytes of fits file
+
         parameter kwargs:
-        fits_id = [str] 
-        
+            fits_id = [int],
+            file_path = [str], 
+            chunk_size = [int] default 20480
+
         yield bytes of fits file
         '''
-        yield self.stub.read(**kwargs)
+        return self.stub.read(**kwargs)
+
+    def update_proc_status(self, **kwargs):
+        ''' update the status of reduction
 
-    def update_status(self, **kwargs):
-        return self.stub.update_status(**kwargs)    
+        parameter kwargs:
+            fits_id = [int],
+            status = [int]
+        '''
+        return self.stub.update_proc_status(**kwargs)
 
-    def upload(self,**kwargs): 
-        yield self.stub.upload(**kwargs)
+    def update_qc0_status(self, **kwargs):
+        ''' update the status of reduction
+        
+        parameter kwargs:
+            fits_id = [int],
+            status = [int]
+        '''        
+        return self.stub.update_qc0_status(**kwargs)    
+
+    def write(self, **kwargs):
+        ''' copy a local file to file storage, then reduce the header of fits file and insert a record into database
+ 
+        parameter kwargs:
+            file_path = [str]
+        '''        
+        yield self.stub.write(**kwargs)
 

csst_dfs_api/ifs/reffits.py

 
+from ..common.delegate import Delegate
+
 class RefFitsApi(object):
-    def __init__(self, sub_system = "ifs"):
+    """
+    Reference Fits Operation Class of IFS
+    """
+    REF_FITS_BIAS = "bias"
+    REF_FITS_FLAT = "flat"
+    REF_FITS_DARK = "dark"
+    REF_FITS_SKY = "sky"
+    REF_FITS_ARC = "arc"
+
+    REF_FITS_TYPES = [REF_FITS_BIAS, REF_FITS_FLAT, REF_FITS_DARK, REF_FITS_SKY, REF_FITS_ARC]
+
+    def __init__(self, sub_system="ifs"):
         self.sub_system = sub_system
-    
-    def fetch(self, **kwargs):
-        pass
+        self.module = Delegate().load(sub_module = "ifs")
+        self.stub = getattr(self.module, "RefFitsApi")()
+        self.file_prefix = self.stub.root_dir
+
+    def find(self, **kwargs):
+        '''
+        parameter kwargs:
+            obs_time = [int],
+            file_name = [str],
+            exp_time = (start, end),
+            status = [int],
+            ref_type = [str]
+
+        return list of reference's files records
+        '''
+        return self.stub.find(**kwargs)
+
+    def get(self, **kwargs):
+        '''  query database, return a record as dict
+
+        parameter kwargs:
+            fits_id = [int] 
+
+        return dict or None
+        '''
+        return self.stub.get(**kwargs)
+
+    def read(self, **kwargs):
+        ''' yield bytes of fits file
+
+        parameter kwargs:
+            fits_id = [int],
+            file_path = [str], 
+            chunk_size = [int] default 20480
+
+        yield bytes of fits file
+        '''
+        return self.stub.read(**kwargs)
+
+    def update_status(self, **kwargs):
+        ''' update the status of reduction
+
+        parameter kwargs:
+            fits_id = [int],
+            status = [int]
+        '''
+        return self.stub.update_status(**kwargs)
+
+    def write(self, **kwargs):
+        ''' copy a local file to file storage, then reduce the header of fits file and insert a record into database
+ 
+        parameter kwargs:
+            file_path = [str]
+        '''        
+        yield self.stub.write(**kwargs)
 

csst_dfs_api/ifs/result0.py

 
+from ..common.delegate import Delegate
+
 class Result0Api(object):
-    def __init__(self, sub_system = "ifs"):
+    """
+    Level 0 Operation Class of IFS
+    """
+    def __init__(self, sub_system="ifs"):
         self.sub_system = sub_system
-    
-    def upload(self, **kwargs):
-        pass
+        self.module = Delegate().load(sub_module = "ifs")
+        self.stub = getattr(self.module, "Result0Api")()
+        self.file_prefix = self.stub.root_dir
+
+    def find(self, **kwargs):
+        '''
+        parameter kwargs:
+            raw_id = [int],
+            file_name = [str],
+            proc_type = [str]
+
+        return list of level 0 record
+        '''
+        return self.stub.find(**kwargs)
+
+    def get(self, **kwargs):
+        '''  query database, return a record as dict
+
+        parameter kwargs:
+            fits_id = [int] 
+
+        return dict or None
+        '''
+        return self.stub.get(**kwargs)
+
+    def read(self, **kwargs):
+        ''' yield bytes of fits file
+
+        parameter kwargs:
+            fits_id = [int],
+            file_path = [str], 
+            chunk_size = [int] default 20480
+
+        yield bytes of fits file
+        '''
+        return self.stub.read(**kwargs)
+
+    def write(self, **kwargs):
+        ''' copy a local level 0 file to file storage, and insert a record into database
+
+        parameter kwargs:
+            raw_id = [int],
+            file_path = [str],
+            proc_type = [str]
+        '''  
+        yield self.stub.write(**kwargs)
+

csst_dfs_api/ifs/result1.py

+
+from ..common.delegate import Delegate
+
+class Result1Api(object):
+    """
+    Level 1 Operation Class of IFS
+    """
+    def __init__(self, sub_system="ifs"):
+        self.sub_system = sub_system
+        self.module = Delegate().load(sub_module = "ifs")
+        self.stub = getattr(self.module, "Result1Api")()
+        self.file_prefix = self.stub.root_dir
+
+    def find(self, **kwargs):
+        '''
+        parameter kwargs:
+            file_name = [str],
+            proc_type = [str]
+
+        return list of level 1 record
+        '''
+        return self.stub.find(**kwargs)
+
+    def get(self, **kwargs):
+        '''  query database, return a record as dict
+
+        parameter kwargs:
+            fits_id = [int] 
+
+        return dict or None
+        '''
+        return self.stub.get(**kwargs)
+
+    def read(self, **kwargs):
+        ''' yield bytes of fits file
+
+        parameter kwargs:
+            fits_id = [int],
+            file_path = [str], 
+            chunk_size = [int] default 20480
+
+        yield bytes of fits file
+        '''
+        return self.stub.read(**kwargs)
+
+    def write(self, **kwargs):
+        ''' copy a local level 1 file to file storage, and insert a record into database
+
+        parameter kwargs:
+            file_path = [str],
+            proc_type = [str],
+            result0_ids = [list]
+        '''  
+        yield self.stub.write(**kwargs)
+

tests/test_ifs_fits.py

+import os
 import unittest
+from astropy.io import fits
 
 from csst_dfs_api.ifs import FitsApi
-
+from csst_dfs_api.common.delegate import Delegate
 class IFSFitsTestCase(unittest.TestCase):
 
     def setUp(self):
         self.api = FitsApi()
 
     def test_find(self):
-        files = self.api.find()
-        print("find ", len(files))
+        recs = self.api.find(file_name='CCD1_ObsTime_300_ObsNum_7.fits')
+        print('find:', recs)
+        assert len(recs) == 1
+
+        recs = self.api.find()
+        print('find:', recs)
+        assert len(recs) > 1
+
+    def test_read(self):
+        recs = self.api.find(file_name='CCD1_ObsTime_300_ObsNum_7.fits')
+        print("The full path: ", os.path.join(self.api.file_prefix, recs[0]['file_path']))
+
+        file_segments = self.api.read(file_path=recs[0]['file_path'])
+        file_bytes = b''.join(file_segments)
+        hdul = fits.HDUList.fromstring(file_bytes)
+        print(hdul.info())
+        hdr = hdul[0].header
+        print(repr(hdr))
+
+    def test_update_proc_status(self):
+        recs = self.api.find(file_name='CCD1_ObsTime_300_ObsNum_7.fits')
+
+        self.api.update_proc_status(fits_id=recs[0]['id'],status=1)
+
+        rec = self.api.get(fits_id=recs[0]['id'])
+        assert rec['prc_status'] == 1
+
+    def test_update_qc0_status(self):
+        recs = self.api.find(file_name='CCD1_ObsTime_300_ObsNum_7.fits')
+
+        self.api.update_qc0_status(fits_id=recs[0]['id'],status=1)
+
+        rec = self.api.get(fits_id=recs[0]['id'])
+        assert rec['qc0_status'] == 1
+
+    def test_write(self):
+        recs = self.api.write(file_path='/opt/temp/csst_ifs/CCD2_ObsTime_1200_ObsNum_40.fits')
+
+        recs = self.api.find(file_name='CCD2_ObsTime_1200_ObsNum_40.fits')
+
+        rec = self.api.get(fits_id=recs[0]['id'])
+
+        print(rec)
     
\ No newline at end of file

tests/test_ifs_refs.py

+import os
+import unittest
+from astropy.io import fits
+
+from csst_dfs_api.ifs import RefFitsApi
+
+class IFSFitsTestCase(unittest.TestCase):
+
+    def setUp(self):
+        self.api = RefFitsApi()
+
+    def test_find(self):
+        recs = self.api.find(exp_time=('2021-03-19 13:42:22', '2021-03-19 15:28:00'), ref_type=RefFitsApi.REF_FITS_FLAT)
+        print('find:', recs)
+        assert len(recs) > 1
+
+        recs = self.api.find()
+        print('=' * 80)
+        print('find:', recs)
+        assert len(recs) > 1
+
+    def test_read(self):
+        recs = self.api.find(file_name='CCD2_Flat_img.fits')
+        print("The full path: ", os.path.join(self.api.file_prefix, recs[0]['file_path']))
+
+        file_segments = self.api.read(file_path=recs[0]['file_path'])
+        file_bytes = b''.join(file_segments)
+        hdul = fits.HDUList.fromstring(file_bytes)
+        print(hdul.info())
+        hdr = hdul[0].header
+        print(repr(hdr))      
+
+    def test_update_status(self):
+        recs = self.api.find(file_name='CCD2_Flat_img.fits')
+        self.api.update_status(fits_id=recs[0]['id'],status=1)
+        rec = self.api.get(fits_id=recs[0]['id'])
+        assert rec['status'] == 1
+
+    def test_write(self):
+        recs = self.api.write(file_path='/opt/temp/csst_ifs/CCD3_Flat_img.fits')
+        recs = self.api.find(file_name='CCD3_Flat_img.fits')
+        rec = self.api.get(fits_id=recs[0]['id'])
+        print(rec)

tests/test_ifs_result0.py

+import os
+import unittest
+from astropy.io import fits
+
+from csst_dfs_api.ifs import Result0Api
+
+class IFSResult0TestCase(unittest.TestCase):
+
+    def setUp(self):
+        self.api = Result0Api()
+
+    def test_find(self):
+        recs = self.api.find(file_name='CCD2_ObsTime_1200_ObsNum_40.fits')
+        print('find:', recs)
+
+        recs = self.api.find()
+        print('find:', recs)
+
+    def test_read(self):
+        recs = self.api.find(file_name='CCD2_ObsTime_1200_ObsNum_40.fits')
+        print("The full path: ", os.path.join(self.api.file_prefix, recs[0]['file_path']))
+
+        file_segments = self.api.read(file_path=recs[0]['file_path'])
+        file_bytes = b''.join(file_segments)
+        hdul = fits.HDUList.fromstring(file_bytes)
+        print(hdul.info())
+        hdr = hdul[0].header
+        print(repr(hdr))      
+
+    def test_write(self):
+        self.api.write(raw_id = 1, 
+                file_path='/opt/temp/csst_ifs/CCD2_ObsTime_1200_ObsNum_40.fits',
+                proc_type = 'default')
+
+        recs = self.api.find(raw_id=1)
+        print()
+        print(recs)
+        print("="*80)
+        recs = self.api.find(file_name='CCD2_ObsTime_1200_ObsNum_40.fits')
+        print(recs)

tests/test_ifs_result1.py

+import os
+import unittest
+from astropy.io import fits
+
+from csst_dfs_api.ifs import Result1Api
+
+class IFSResult1TestCase(unittest.TestCase):
+
+    def setUp(self):
+        self.api = Result1Api()
+
+    def test_find(self):
+        recs = self.api.find(file_name='CCD2_ObsTime_1200_ObsNum_40.fits')
+        print('find:', recs)
+
+        recs = self.api.find()
+        print('find:', recs)
+
+    def test_read(self):
+        recs = self.api.find(file_name='CCD2_ObsTime_1200_ObsNum_40.fits')
+        print("The full path: ", os.path.join(self.api.file_prefix, recs[0]['file_path']))
+
+        rec, result0s = self.api.get(fits_id=recs[0]['id'])
+        print(result0s)
+        file_segments = self.api.read(file_path=rec['file_path'])
+        file_bytes = b''.join(file_segments)
+        hdul = fits.HDUList.fromstring(file_bytes)
+        print(hdul.info())
+        hdr = hdul[0].header
+        print(repr(hdr))
+
+    def test_write(self):
+        self.api.write(result0_ids = [1,2,3,4], 
+                file_path='/opt/temp/csst_ifs/CCD2_ObsTime_1200_ObsNum_40.fits',
+                proc_type = 'default')
+
+        recs = self.api.find(raw_id=1)
+        print()
+        print(recs)
+        print("="*80)
+        recs = self.api.find(file_name='CCD2_ObsTime_1200_ObsNum_40.fits')
+        print(recs)