split level0data

csst_dfs_api_local/sls/level1prc.py

+import os
+import logging
+import time, datetime
+import shutil
+
+from ..common.db import DBClient
+from ..common.utils import *
+from csst_dfs_commons.models import Result
+from csst_dfs_commons.models.sls import Level1PrcRecord
+from csst_dfs_commons.models.common import from_dict_list
+
+log = logging.getLogger('csst')
+
+class Level1PrcApi(object):
+    def __init__(self, sub_system = "sls"):
+        self.sub_system = sub_system
+        self.root_dir = os.getenv("CSST_LOCAL_FILE_ROOT", "/opt/temp/csst")
+        self.db = DBClient()
+
+    def find(self, **kwargs):
+        ''' retrieve level1 procedure records from database
+
+        parameter kwargs:
+            level1_id: [int]
+            pipeline_id: [str]
+            prc_module: [str]
+            prc_status : [int]
+
+        return: csst_dfs_common.models.Result
+        '''
+        try:
+            level1_id = get_parameter(kwargs, "level1_id", 0)
+            pipeline_id = get_parameter(kwargs, "pipeline_id")
+            prc_module = get_parameter(kwargs, "prc_module")
+            prc_status = get_parameter(kwargs, "prc_status")
+
+            sql_data = f"select * from sls_level1_prc"
+
+            sql_condition = f"where level1_id={level1_id}"
+            if pipeline_id:
+                sql_condition = sql_condition + " and pipeline_id='" + pipeline_id + "'"
+            if prc_module:
+                sql_condition = sql_condition + " and prc_module ='" + prc_module + "'"
+            if prc_status:
+                sql_condition = f"{sql_condition} and prc_status={prc_status}"   
+
+            sql_data = f"{sql_data} {sql_condition}"
+
+            _, records = self.db.select_many(sql_data)
+            return Result.ok_data(data=from_dict_list(Level1PrcRecord, records)).append("totalCount", len(records))
+
+        except Exception as e:
+            return Result.error(message=str(e))
+
+    def update_proc_status(self, **kwargs):
+        ''' update the status of reduction
+
+        parameter kwargs:
+            id : [int],
+            status : [int]
+
+        return csst_dfs_common.models.Result
+        '''
+        id = get_parameter(kwargs, "id")
+        status = get_parameter(kwargs, "status")
+
+        try:
+            existed = self.db.exists(
+                "select * from sls_level1_prc where id=?",
+                (id,)
+            )
+            if not existed:
+                log.warning('%s not found' %(id, ))
+                return Result.error(message ='%s not found' %(id, ))
+            self.db.execute(
+                'update sls_level1_prc set prc_status=?, prc_time=? where id=?',
+                (status, format_time_ms(time.time()), id)
+            )  
+            self.db.end() 
+            return Result.ok_data()
+           
+        except Exception as e:
+            log.error(e)
+            return Result.error(message=str(e))
+
+    def write(self, **kwargs):
+        ''' insert a level1 procedure record into database
+ 
+        parameter kwargs:
+            level1_id : [int]
+            pipeline_id : [str]
+            prc_module : [str]
+            params_file_path : [str]
+            prc_status : [int]
+            prc_time : [str]
+            result_file_path : [str]
+        return csst_dfs_common.models.Result
+        '''   
+
+        rec = Level1PrcRecord(
+            id = 0,
+            level1_id = get_parameter(kwargs, "level1_id", 0),
+            pipeline_id = get_parameter(kwargs, "pipeline_id"),
+            prc_module = get_parameter(kwargs, "prc_module"),
+            params_file_path = get_parameter(kwargs, "params_file_path"),
+            prc_status = get_parameter(kwargs, "prc_status", -1),
+            prc_time = get_parameter(kwargs, "prc_time"),
+            result_file_path = get_parameter(kwargs, "result_file_path")
+        )
+        try:
+            self.db.execute(
+                'INSERT INTO sls_level1_prc (level1_id,pipeline_id,prc_module, params_file_path, prc_status,prc_time,result_file_path) \
+                    VALUES(?,?,?,?,?,?,?)',
+                (rec.level1_id, rec.pipeline_id, rec.prc_module, rec.params_file_path, rec.prc_status, rec.prc_time, rec.result_file_path)
+            )
+            self.db.end()
+            rec.id = self.db.last_row_id()
+
+            return Result.ok_data(data=rec)
+
+        except Exception as e:
+            log.error(e)
+            return Result.error(message=str(e)) 
+

csst_dfs_api_local/sls/level2spectra.py

@@ -12,7 +12,7 @@ from csst_dfs_commons.models.common import from_dict_list
 log = logging.getLogger('csst')
 
 class Level2SpectraApi(object):
-    def __init__(self, sub_system = "msc"):
+    def __init__(self, sub_system = "sls"):
         self.sub_system = sub_system
         self.root_dir = os.getenv("CSST_LOCAL_FILE_ROOT", "/opt/temp/csst")
         self.db = DBClient()

setup.cfg

@@ -27,4 +27,5 @@ install_requires =
 csst_dfs_api_local.common = *.sql
 [options.entry_points]
 console_scripts =
-    csst-dfs-ingest-local = csst_dfs_api_local.common.ingest:ingest
\ No newline at end of file
+    csst-msc-ingest-local = csst_dfs_api_local.msc.ingest:ingest
+    csst-ifs-ingest-local = csst_dfs_api_local.ifs.ingest:ingest
\ No newline at end of file

tests/test_facility_cal_merge.py → tests/test_ifs_cal_merge.py

@@ -2,9 +2,9 @@ import os
 import unittest
 from astropy.io import fits
 
-from csst_dfs_api_local.facility.calmerge import CalMergeApi
+from csst_dfs_api_local.ifs.calmerge import CalMergeApi
 
-class CalMergeApiTestCase(unittest.TestCase):
+class IFSCalMergeApiTestCase(unittest.TestCase):
 
     def setUp(self):
         self.api = CalMergeApi()
@@ -24,11 +24,11 @@ class CalMergeApiTestCase(unittest.TestCase):
         print('get:', rec)
 
     def test_update_proc_status(self):
-        rec = self.api.update_proc_status(id = 1, status = 1)
+        rec = self.api.update_proc_status(id = 100, status = 1)
         print('update_proc_status:', rec)
 
     def test_update_qc1_status(self):
-        rec = self.api.update_qc1_status(id = 1, status = 2)
+        rec = self.api.update_qc1_status(id = 100, status = 2)
         print('update_qc1_status:', rec)    
 
     def test_write(self):

tests/test_ifs_ingest.py

+import unittest
+
+from csst_dfs_api_local.ifs.ingest import ingest, ingesst_one
+
+class IFSLevel0DataApiTestCase(unittest.TestCase):
+
+    def setUp(self):
+        pass
+
+    def ingesst_one(self):
+        ingesst_one()
+
+    def ingest(self):
+        ingest

tests/test_facility_level0.py → tests/test_ifs_level0.py

 import unittest
 
-from csst_dfs_api_local.facility import Level0DataApi
+from csst_dfs_api_local.ifs import Level0DataApi
 
-class Level0DataApiTestCase(unittest.TestCase):
+class IFSLevel0DataApiTestCase(unittest.TestCase):
 
     def setUp(self):
         self.api = Level0DataApi()
 
     def test_find(self):
-        recs = self.api.find(obs_id = '13', obs_type = 'sci', limit = 0)
+        recs = self.api.find(obs_id = '300000055', obs_type = 'sky', limit = 0)
         print('find:', recs)
 
     def test_get(self):
@@ -27,7 +27,7 @@ class Level0DataApiTestCase(unittest.TestCase):
         rec = self.api.write(
             obs_id = '0000013',
             detector_no = "01",
-            obs_type = "sci",            
+            obs_type = "sky",            
             obs_time = "2021-06-06 11:12:13",
             exp_time = 150,
             detector_status_id = 3,

tests/test_facility_level0_prc.py → tests/test_ifs_level0_prc.py

@@ -2,15 +2,15 @@ import os
 import unittest
 from astropy.io import fits
 
-from csst_dfs_api_local.facility.level0prc import Level0PrcApi
+from csst_dfs_api_local.ifs.level0prc import Level0PrcApi
 
-class Level0PrcTestCase(unittest.TestCase):
+class IFSLevel0PrcTestCase(unittest.TestCase):
 
     def setUp(self):
         self.api = Level0PrcApi()
 
     def test_find(self):
-        recs = self.api.find(level0_id='134')
+        recs = self.api.find(level0_id='300000055CCD231-c4')
         print('find:', recs)
 
     def test_update_proc_status(self):
@@ -18,7 +18,7 @@ class Level0PrcTestCase(unittest.TestCase):
         print('update_proc_status:', rec)
 
     def test_write(self):
-        rec = self.api.write(level0_id='134', 
+        rec = self.api.write(level0_id='300000055CCD231-c4', 
             pipeline_id = "P1",
             prc_module = "QC0",
             params_file_path = "/opt/dddasd.params",

tests/test_ifs_level1.py

@@ -2,7 +2,7 @@ import unittest
 
 from csst_dfs_api_local.ifs import Level1DataApi
 
-class IFSResult1TestCase(unittest.TestCase):
+class IFSLevel1DataApiTestCase(unittest.TestCase):
 
     def setUp(self):
         self.api = Level1DataApi()

tests/test_facility_level1_prc.py → tests/test_ifs_level1_prc.py

@@ -2,9 +2,9 @@ import os
 import unittest
 from astropy.io import fits
 
-from csst_dfs_api_local.facility.level1prc import Level1PrcApi
+from csst_dfs_api_local.ifs.level1prc import Level1PrcApi
 
-class Level1PrcTestCase(unittest.TestCase):
+class IFSLevel1PrcTestCase(unittest.TestCase):
 
     def setUp(self):
         self.api = Level1PrcApi()

tests/test_msc_cal_merge.py

+import os
+import unittest
+from astropy.io import fits
+
+from csst_dfs_api_local.msc.calmerge import CalMergeApi
+
+class MSCCalMergeApiTestCase(unittest.TestCase):
+
+    def setUp(self):
+        self.api = CalMergeApi()
+
+    def test_find(self):
+        recs = self.api.find(detector_no='01',
+            ref_type = "bias",
+            obs_time = ("2021-06-01 11:12:13","2021-06-08 11:12:13"))
+        print('find:', recs)
+
+    def test_get_latest_by_l0(self):
+        rec = self.api.get_latest_by_l0(level0_id='000001301', ref_type = "bias")
+        print('get:', rec)
+
+    def test_get(self):
+        rec = self.api.get(cal_id='0000231')
+        print('get:', rec)
+
+    def test_update_proc_status(self):
+        rec = self.api.update_proc_status(id = 100, status = 1)
+        print('update_proc_status:', rec)
+
+    def test_update_qc1_status(self):
+        rec = self.api.update_qc1_status(id = 100, status = 2)
+        print('update_qc1_status:', rec)    
+
+    def test_write(self):
+        rec = self.api.write(
+            cal_id='0000231',
+            detector_no='01', 
+            ref_type = "bias",
+            obs_time = "2021-06-04 11:12:13",
+            exp_time = 150,
+            filename = "/opt/dddasd.params",
+            file_path = "/opt/dddasd.fits",
+            prc_status = 3,
+            prc_time = '2021-06-04 11:12:13',
+            level0_ids = ['0000231','0000232','0000233','0000234'])
+        print('write:', rec)
\ No newline at end of file

tests/test_msc_level0.py

+import unittest
+
+from csst_dfs_api_local.msc import Level0DataApi
+
+class MSCLevel0DataApiTestCase(unittest.TestCase):
+
+    def setUp(self):
+        self.api = Level0DataApi()
+
+    def test_find(self):
+        recs = self.api.find(obs_id = '300000055', obs_type = 'sky', limit = 0)
+        print('find:', recs)
+
+    def test_get(self):
+        rec = self.api.get(id = 31)
+        print('get:', rec)
+
+    def test_update_proc_status(self):
+        rec = self.api.update_proc_status(id = 31, status = 6)
+        print('update_proc_status:', rec)
+
+    def test_update_qc0_status(self):
+        rec = self.api.update_qc0_status(id = 31, status = 7)
+        print('update_qc0_status:', rec)
+
+    def test_write(self):
+        rec = self.api.write(
+            obs_id = '0000013',
+            detector_no = "01",
+            obs_type = "sky",            
+            obs_time = "2021-06-06 11:12:13",
+            exp_time = 150,
+            detector_status_id = 3,
+            filename = "MSC_00001234",
+            file_path = "/opt/MSC_00001234.fits")
+        print('write:', rec)  

tests/test_msc_level0_prc.py

+import os
+import unittest
+from astropy.io import fits
+
+from csst_dfs_api_local.msc.level0prc import Level0PrcApi
+
+class MSCLevel0PrcTestCase(unittest.TestCase):
+
+    def setUp(self):
+        self.api = Level0PrcApi()
+
+    def test_find(self):
+        recs = self.api.find(level0_id='300000055CCD231-c4')
+        print('find:', recs)
+
+    def test_update_proc_status(self):
+        rec = self.api.update_proc_status(id = 1, status = 4)
+        print('update_proc_status:', rec)
+
+    def test_write(self):
+        rec = self.api.write(level0_id='300000055CCD231-c4', 
+            pipeline_id = "P1",
+            prc_module = "QC0",
+            params_file_path = "/opt/dddasd.params",
+            prc_status = 3,
+            prc_time = '2021-06-04 11:12:13',
+            result_file_path = "/opt/dddasd.header")
+        print('write:', rec)
\ No newline at end of file

tests/test_msc_level1.py

@@ -2,7 +2,7 @@ import unittest
 
 from csst_dfs_api_local.msc import Level1DataApi
 
-class IFSResult1TestCase(unittest.TestCase):
+class MSCLevel1DataTestCase(unittest.TestCase):
 
     def setUp(self):
         self.api = Level1DataApi()

tests/test_msc_level1_prc.py

+import os
+import unittest
+from astropy.io import fits
+
+from csst_dfs_api_local.msc.level1prc import Level1PrcApi
+
+class MSCLevel1PrcTestCase(unittest.TestCase):
+
+    def setUp(self):
+        self.api = Level1PrcApi()
+
+    def test_find(self):
+        recs = self.api.find(level1_id=1)
+        print('find:', recs)
+
+    def test_update_proc_status(self):
+        rec = self.api.update_proc_status(id = 1, status = 4)
+        print('update_proc_status:', rec)
+
+    def test_write(self):
+        rec = self.api.write(level1_id=1, 
+            pipeline_id = "P1",
+            prc_module = "QC0",
+            params_file_path = "/opt/dddasd.params",
+            prc_status = 3,
+            prc_time = '2021-06-04 11:12:13',
+            result_file_path = "/opt/dddasd.header")
+        print('write:', rec)
\ No newline at end of file

tests/test_msc_level2_catalog.py

+import os
+import unittest
+from astropy.io import fits
+
+from csst_dfs_api_local.msc.level2catalog import Level2CatalogApi
+
+class MSCLevel2CatalogApiTestCase(unittest.TestCase):
+
+    def setUp(self):
+        self.api = Level2CatalogApi()
+
+    def test_find(self):
+        recs = self.api.find(
+            obs_id='100000000',
+            obs_time = ("2021-05-24 11:12:13","2021-05-25 13:12:13"),
+            limit = 100)
+        print('find:', recs)
+
+    def test_write(self):
+        rec = self.api.write([])
+        print('write:', rec)
\ No newline at end of file

tests/test_sls_cal_merge.py

+import os
+import unittest
+from astropy.io import fits
+
+from csst_dfs_api_local.sls.calmerge import CalMergeApi
+
+class SLSCalMergeApiTestCase(unittest.TestCase):
+
+    def setUp(self):
+        self.api = CalMergeApi()
+
+    def test_find(self):
+        recs = self.api.find(detector_no='01',
+            ref_type = "bias",
+            obs_time = ("2021-06-01 11:12:13","2021-06-08 11:12:13"))
+        print('find:', recs)
+
+    def test_get_latest_by_l0(self):
+        rec = self.api.get_latest_by_l0(level0_id='000001301', ref_type = "bias")
+        print('get:', rec)
+
+    def test_get(self):
+        rec = self.api.get(cal_id='0000231')
+        print('get:', rec)
+
+    def test_update_proc_status(self):
+        rec = self.api.update_proc_status(id = 100, status = 1)
+        print('update_proc_status:', rec)
+
+    def test_update_qc1_status(self):
+        rec = self.api.update_qc1_status(id = 100, status = 2)
+        print('update_qc1_status:', rec)    
+
+    def test_write(self):
+        rec = self.api.write(
+            cal_id='0000231',
+            detector_no='01', 
+            ref_type = "bias",
+            obs_time = "2021-06-04 11:12:13",
+            exp_time = 150,
+            filename = "/opt/dddasd.params",
+            file_path = "/opt/dddasd.fits",
+            prc_status = 3,
+            prc_time = '2021-06-04 11:12:13',
+            level0_ids = ['0000231','0000232','0000233','0000234'])
+        print('write:', rec)
\ No newline at end of file

tests/test_sls_level0.py

+import unittest
+
+from csst_dfs_api_local.sls import Level0DataApi
+
+class SLSLevel0DataApiTestCase(unittest.TestCase):
+
+    def setUp(self):
+        self.api = Level0DataApi()
+
+    def test_find(self):
+        recs = self.api.find(obs_id = '300000055', obs_type = 'sky', limit = 0)
+        print('find:', recs)
+
+    def test_get(self):
+        rec = self.api.get(id = 31)
+        print('get:', rec)
+
+    def test_update_proc_status(self):
+        rec = self.api.update_proc_status(id = 31, status = 6)
+        print('update_proc_status:', rec)
+
+    def test_update_qc0_status(self):
+        rec = self.api.update_qc0_status(id = 31, status = 7)
+        print('update_qc0_status:', rec)
+
+    def test_write(self):
+        rec = self.api.write(
+            obs_id = '0000013',
+            detector_no = "01",
+            obs_type = "sky",            
+            obs_time = "2021-06-06 11:12:13",
+            exp_time = 150,
+            detector_status_id = 3,
+            filename = "MSC_00001234",
+            file_path = "/opt/MSC_00001234.fits")
+        print('write:', rec)  

tests/test_sls_level0_prc.py

+import os
+import unittest
+from astropy.io import fits
+
+from csst_dfs_api_local.sls.level0prc import Level0PrcApi
+
+class SLSLevel0PrcTestCase(unittest.TestCase):
+
+    def setUp(self):
+        self.api = Level0PrcApi()
+
+    def test_find(self):
+        recs = self.api.find(level0_id='300000055CCD231-c4')
+        print('find:', recs)
+
+    def test_update_proc_status(self):
+        rec = self.api.update_proc_status(id = 1, status = 4)
+        print('update_proc_status:', rec)
+
+    def test_write(self):
+        rec = self.api.write(level0_id='300000055CCD231-c4', 
+            pipeline_id = "P1",
+            prc_module = "QC0",
+            params_file_path = "/opt/dddasd.params",
+            prc_status = 3,
+            prc_time = '2021-06-04 11:12:13',
+            result_file_path = "/opt/dddasd.header")
+        print('write:', rec)
\ No newline at end of file

tests/test_sls_level1_prc.py

+import os
+import unittest
+from astropy.io import fits
+
+from csst_dfs_api_local.sls.level1prc import Level1PrcApi
+
+class SLSLevel1PrcTestCase(unittest.TestCase):
+
+    def setUp(self):
+        self.api = Level1PrcApi()
+
+    def test_find(self):
+        recs = self.api.find(level1_id=1)
+        print('find:', recs)
+
+    def test_update_proc_status(self):
+        rec = self.api.update_proc_status(id = 1, status = 4)
+        print('update_proc_status:', rec)
+
+    def test_write(self):
+        rec = self.api.write(level1_id=1, 
+            pipeline_id = "P1",
+            prc_module = "QC0",
+            params_file_path = "/opt/dddasd.params",
+            prc_status = 3,
+            prc_time = '2021-06-04 11:12:13',
+            result_file_path = "/opt/dddasd.header")
+        print('write:', rec)
\ No newline at end of file