Update RSS_demo

csst_dfs_api_local/common/db.py

@@ -45,7 +45,10 @@ class DBClient(object):
         count = self.__execute(sql, param)
         result = self._cursor.fetchone()
         """:type result:dict"""
-        result = self.__dict_datetime_obj_to_str(result)
+        # result = self.__dict_datetime_obj_to_str(result)
+        result = {
+            key[0]: col for key, col in zip(self._cursor.description, result)
+        }
         return count, result
 
     def select_many(self, sql, param=()):
@@ -58,8 +61,14 @@ class DBClient(object):
         count = self.__execute(sql, param)
         result = self._cursor.fetchall()
         """:type result:list"""
-        [self.__dict_datetime_obj_to_str(row_dict) for row_dict in result]
-        return count, result
+        # result = [self.__dict_datetime_obj_to_str(row_dict) for row_dict in result]
+        r = []
+        for row_dict in result:
+            r.append(
+                {key[0]:col for key, col in zip(self._cursor.description, row_dict)}
+            )
+            
+        return count, r
 
     def execute(self, sql, param=()):
         count = self.__execute(sql, param)

csst_dfs_api_local/entity/RSS_demo.py

+import os
+
+import numpy as np
+import pandas as pd
+from csst_dfs_api_local.ifs import FitsApi
+
+
+class RSS():
+
+    def __init__(self, rawdata):
+        self.fitsapi = FitsApi()
+        try:
+            self.raw = self.fitsapi.find(fits_id=rawdata)
+            if not os.path.exists(self.raw):
+                print(self.raw, 'does not exist')
+                self.raw = False
+        except:
+            print(rawdata, 'not in database')
+            self.raw = False
+
+    def set_bias(self, bias_file=None):
+        try:
+            self.bias = self.fitsapi.find(fits_id=bias_file)
+            if not os.path.exists(self.bias):
+                print(self.bias, 'does not exist')
+                self.bias = False
+        except:
+            print(bias_file, 'not in database')
+            self.bias = False
+
+    def set_flat(self, flat_file=None):
+        try:
+            self.flat = self.fitsapi.find(fits_id=flat_file)
+            if not os.path.exists(self.flat):
+                print(self.flat, 'does not exist')
+                self.flat = False
+        except:
+            print(flat_file, 'not in database')
+            self.flat = False
+
+    def set_arc(self, arc_file=None):
+        try:
+            self.arc = self.fitsapi.find(fits_id=arc_file)
+            if not os.path.exists(self.arc):
+                print(self.arc, 'does not exist')
+                self.arc = False
+        except:
+            print(arc_file, 'not in database')
+            self.arc = False
+
+    def set_sky(self, sky_file=None):
+        try:
+            self.sky = self.fitsapi.find(fits_id=sky_file)
+            if not os.path.exists(self.sky):
+                print(self.sky, 'does not exist')
+                self.sky = False
+        except:
+            print(sky_file, 'not in database')
+            self.sky = False
+
+    def makecube(self, outfile):
+
+        refiles = [self.raw, self.arc, self.flat, self.bias, self.sky]
+        print('reference files: ', refiles)
+        df = pd.DataFrame(refiles)
+
+        df.to_pickle(outfile)
+
+
+if __name__ == '__main__':
+
+    rss1 = RSS('CCD1_ObsTime_600_ObsNum_30.fits')     # raw data
+
+    rss1.bias()                                     # currently no Bias file
+    rss1.flat(flat_file='Flat_flux.fits')           # flat file
+    rss1.arc(arc_file='HgAr_flux.fits')             # arc file
+    rss1.sky(sky_file='sky_noise_With_wavelength.fits')  # sky file
+
+    rss1.makecube('rss_demo.pkl')

csst_dfs_api_local/entity/ReadMe.txt

+RSS_demo
+
+输入数据：
+
+（1）原始数据
+文件名：CCD1_ObsTime_600_ObsNum_30.fits  
+说明：原始数据文件名不能为None；按照文件名找不到对应文件的时候，不报错，设置该文件状态为False。
+
+（2）参考文件
+
+平场参考文件： Flat_flux.fits
+灯谱文件：HgAr_flux.fits
+天光背景文件：sky_noise_With_wavelength.fits
+说明：参考文件名可以为None，比如程序中的bias_file=None；文件名不为None，但是找不到对应文件的时候，不报错，设置该文件状态为False。
+
+
+输出数据：
+
+rss_demo.pkl （记录原始数据和四个参考文件的状态）

csst_dfs_api_local/entity/__init__.py

+from .RSS_demo import RSS
\ No newline at end of file

csst_dfs_api_local/ifs/fits.py

@@ -22,12 +22,12 @@ class FitsApi(object):
         if not os.path.exists(self.root_dir):
             os.mkdir(self.root_dir)
             log.info("using [%s] as root directory", self.root_dir)
-        if not os.path.exists(os.path.join(self.root_dir, "fits")):
-            os.mkdir(os.path.join(self.root_dir, "fits"))
-        if not os.path.exists(os.path.join(self.root_dir, "refs")):
-            os.mkdir(os.path.join(self.root_dir, "refs"))
-        if not os.path.exists(os.path.join(self.root_dir, "results")):
-            os.mkdir(os.path.join(self.root_dir, "results"))
+        # if not os.path.exists(os.path.join(self.root_dir, "fits")):
+        #     os.mkdir(os.path.join(self.root_dir, "fits"))
+        # if not os.path.exists(os.path.join(self.root_dir, "refs")):
+        #     os.mkdir(os.path.join(self.root_dir, "refs"))
+        # if not os.path.exists(os.path.join(self.root_dir, "results")):
+        #     os.mkdir(os.path.join(self.root_dir, "results"))
 
     def find(self, **kwargs):
         '''
@@ -39,6 +39,7 @@ class FitsApi(object):
         return list of paths
         '''
         paths = []
+        
         obs_time = get_parameter(kwargs, "obs_time")
         type = get_parameter(kwargs, "type")
         fits_id = get_parameter(kwargs, "fits_id")
@@ -54,7 +55,15 @@ class FitsApi(object):
             if len(r) < 1:
                 raise Exception('not found')
             for items in r:
-                paths.append(items[4])
+                paths.append(os.path.join(self.root_dir, items['path']))
+        else:
+            c, r = self.db.select_many(
+                'select * from t_rawfits where id=?',
+                (fits_id,),
+            )
+            if len(r) < 1:
+                raise Exception('not found')
+            return os.path.join(self.root_dir, r[0]['path'])
         return paths
 
     def read(self, **kwargs):
@@ -76,11 +85,12 @@ class FitsApi(object):
             c, r = self.db.select_one(
                 "select * from t_rawfits where id=?", (fits_id))
             if c == 1:
-                file_path = r["path"]
+                file_path = os.path.join(self.root_dir, r["path"])
 
         if file_path is not None:
+            path = os.path.join(self.root_dir, file_path)
             chunk_size = get_parameter(kwargs, "chunk_size", 1024)
-            with open(file_path, 'r') as f:
+            with open(path, 'r') as f:
                 while True:
                     data = f.read(chunk_size)
                     if not data:
@@ -103,37 +113,46 @@ class FitsApi(object):
             raise Exception("file_path need to be defined")
 
         basename = os.path.basename(file_path)
-        name = basename.split('.fits')[0]
+        name = basename.split('.fits')[0].lower()
         c, r = self.db.select_many(
             "select * from t_rawfits where id=?",
             (name,)
         )
         if len(r) >= 1:
-            print('already upload', name)
+            print('already upload', basename)
             return
 
-        hu = fits.getheader(file_path)
-        obs_time = hu['obst']
-        ccd_num = hu['ccd_num']
+        hu = fits.getheader(os.path.join(self.root_dir, file_path))
+        obs_time = hu['obst'] if 'obst' in hu else 0
+        ccd_num = hu['ccd_num'] if 'ccd_num' in hu else 0
         # print(obs_time, ccd_num)
-        type = 'obs'
-        save_path = os.path.join(self.root_dir, 'fits')
-        save_path = os.path.join(save_path, basename)
+        if 'obs' in name:
+            type = 'obs'
+        elif 'flat' in name:
+            type = 'flat'
+        elif 'bias' in name:
+            type = 'bias'
+        elif 'hgar' in name:
+            type = 'arc'
+        elif 'sky' in name:
+            type = 'sky'
+        else:
+            type = 'None'
+        
 
         self.db.execute(
             'INSERT INTO t_rawfits VALUES(?,?,?,?,?)',
-            (name, obs_time, ccd_num, type, save_path)
+            (basename, obs_time, ccd_num, type, file_path)
         )
         self.db._conn.commit()
-        if file_path != save_path:
-            shutil.copyfile(file_path, save_path)
-        log.info("%s imported.", save_path)
+        log.info("%s imported.", file_path)
 
     def scan2db(self):
         paths = {}
 
-        for (path, _, file_names) in os.walk(os.path.join(self.root_dir, "fits")):
+        for (path, _, file_names) in os.walk(self.root_dir):
             for filename in file_names:
                 if filename.find(".fits") > 0:
-                    self.upload(file_path=os.path.join(path, filename))
+                    filename = os.path.join(path, filename)
+                    self.upload(file_path=filename.replace(self.root_dir, ''))
         return paths

csst_dfs_api_local/ifs/reffits.py

@@ -43,7 +43,7 @@ class RefFitsApi(FitsApi):
 
         self.db.execute(
             'INSERT INTO t_rawfits VALUES(?,?,?,?,?)',
-            (name, obs_time, ccd_num, type, save_path)
+            (basename, obs_time, ccd_num, type, save_path)
         )
         self.db._conn.commit()
         if file_path != save_path:

tests/test.py

+import logging
+
+from csst_dfs_api_local.ifs import FitsApi
+
+api = FitsApi()
+# api.scan2db()
+c, r = api.db.select_one(
+    "select * from t_rawfits where id=?",
+    ('CCD1_Flat_img.fits', )
+)
+print(r)
\ No newline at end of file

tests/test_RSS.py

+import logging
+import unittest
+import os
+
+from csst_dfs_api_local.entity import RSS
+
+log = logging.getLogger('csst')
+class RSS_TestCase(unittest.TestCase):
+    def setUp(self):
+        self.rss = RSS('CCD1_ObsTime_600_ObsNum_30.fits')
+        self.rss.set_bias()
+        self.rss.set_flat(flat_file='Flat_flux.fits')
+        self.rss.set_arc(arc_file='HgAr_flux.fits')
+        self.rss.set_sky(sky_file='sky_noise_With_wavelength.fits')
+
+    def test_init(self):
+        assert self.rss.raw
+    
+    # def test_bias(self):
+    #     self.rss.set_bias()
+    #     assert self.rss.bias
+
+    def test_flat(self):
+        assert self.rss.flat
+
+    def test_arc(self):
+        assert self.rss.arc
+
+    def test_sky(self):
+        assert self.rss.sky
+
+    def test_makecube(self):
+        self.rss.makecube('rss_demo.pkl')
+        assert os.path.exists('rss_demo.pkl')
+        os.remove('rss_demo.pkl')
\ No newline at end of file

tests/test_ifs_fits.py

 import logging
 import unittest
+import os
 
 from csst_dfs_api_local.ifs import FitsApi
 
@@ -8,16 +9,19 @@ class IFSFitsTestCase(unittest.TestCase):
 
     def setUp(self):
         self.api = FitsApi()
-        self.api.scan2db()
+        # self.api.scan2db()
 
     def test_find(self):
-        path = self.api.find(obs_time=900, type='obs')
-        log.info('find', path)
-        path = self.api.find(fits_id='CCD2_ObsTime_600_ObsNum_8')
-        log.info('find', path)
+        path1 = self.api.find(obs_time=900, type='obs')
+        log.info('find', path1)
+        assert len(path1) > 0
+        path2 = self.api.find(fits_id='CCD2_ObsTime_600_ObsNum_8.fits')
+        log.info('find', path2)
+        assert 'CCD2_ObsTime_600_ObsNum_8.fits' in path2
+
     
     def test_read(self):
-        file = self.api.read(fits_id='CCD2_ObsTime_600_ObsNum_8')
+        file = self.api.read(fits_id='CCD2_ObsTime_600_ObsNum_8.fits')
         log.info('read', str(type(file)))
         path = self.api.find(obs_time=900, type='obs')
         file = self.api.read(file_path=path)

tests/test_ifs_refs.py

-import logging
-import unittest
+# import logging
+# import unittest
 
-from csst_dfs_api_local.ifs import RefFitsApi
+# from csst_dfs_api_local.ifs import RefFitsApi
 
-log = logging.getLogger('csst')
-class IFSFitsTestCase(unittest.TestCase):
+# log = logging.getLogger('csst')
+# class IFSFitsTestCase(unittest.TestCase):
 
-    def setUp(self):
-        self.api = RefFitsApi()
-        self.api.scan2db()
+#     def setUp(self):
+#         self.api = RefFitsApi()
+#         self.api.scan2db()
 
-    def test_find(self):
-        path = self.api.find(obs_time=300, type='Flat')
-        log.info('find', path)
-        path = self.api.find(fits_id='CCD1_Flat_img')
-        log.info('find', path)
+#     def test_find(self):
+#         path = self.api.find(obs_time=300, type='Flat')
+#         log.info('find', path)
+#         path = self.api.find(fits_id='CCD1_Flat_img.fits')
+#         print(path)
+#         log.info('find', path)
     
-    def test_read(self):
-        file = self.api.read(fits_id='CCD1_Flat_img')
-        log.info('read', str(type(file)))
-        path = self.api.find(obs_time=300, type='Flat')
-        file = self.api.read(file_path=path)
-        log.info('read', str(type(file)))
\ No newline at end of file
+#     def test_read(self):
+#         file = self.api.read(fits_id='CCD1_Flat_img.fits')
+#         log.info('read', str(type(file)))
+#         path = self.api.find(obs_time=300, type='Flat')
+#         file = self.api.read(file_path=path)
+#         log.info('read', str(type(file)))