ifs ver

csst_dfs_api_local/common/db.sql

@@ -31,6 +31,12 @@ drop table if exists msc_level1_header;
 drop table if exists msc_level1_prc;
 
 drop table if exists msc_level1_ref;
+
+drop table if exists msc_level2_data;
+
+drop table if exists msc_level2_header;
+
+drop table if exists msc_level2_catalog;
 /*----------------ifs------------------------------*/
 
 drop table if exists ifs_level0_data;
@@ -293,6 +299,26 @@ create table msc_level1_prc
    prc_time             datetime,
    result_file_path            varchar(256)
 );
+
+create table msc_level2_data (
+   id integer PRIMARY KEY autoincrement,
+   level1_id            int(20)          not null,
+   data_type            VARCHAR(64)          not null,
+   filename             VARCHAR(128)         null,
+   file_path            VARCHAR(256)         null,
+   prc_status           tinyint(1),
+   prc_time             datetime                 null,
+   qc2_status           tinyint(1),
+   qc2_time             datetime                 null,
+   create_time          datetime                 null
+);
+
+create table msc_level2_header (
+   id                   BIGINT                 not null,
+   ra                   float8               null,
+   "dec"                float8               null,
+   constraint PK_MSC_LEVEL2_HEADER primary key (id)
+);
 /*===========================ifs===================================*/
 create table ifs_level0_data
 (
@@ -300,7 +326,6 @@ create table ifs_level0_data
    level0_id            varchar(20) not null,
    obs_id               varchar(10) not null,
    detector_no          varchar(10) not null,
-   object_name          varchar(64) not null,
    obs_type             varchar(16),
    obs_time             datetime,
    exp_time             float,
@@ -317,11 +342,10 @@ create table ifs_level0_data
 create table ifs_level0_header
 (
    id                   int(20) not null,
-   obs_time             datetime,
-   exp_time             float,
    ra                   float,
    "dec"                float,
-   create_time          datetime,
+   object_name          varchar(64) not null,
+   version              varchar(64) not null,
    primary key (id)
 );
 
@@ -439,11 +463,10 @@ create table mci_level0_data
 create table mci_level0_header
 (
    id                   int(20) not null,
-   obs_time             datetime,
-   exp_time             float,
    ra                   float,
    "dec"                float,
-   create_time          datetime,
+   object_name          varchar(64) not null,
+   version              varchar(64) not null,
    primary key (id)
 );
 
@@ -468,8 +491,6 @@ create table mci_cal2level0
 create table mci_cal_header
 (
    id                   int(20) not null,
-   obs_time             datetime,
-   exp_time             float,
    ra                   float,
    "dec"                float,
    create_time          datetime,
@@ -686,12 +707,9 @@ create table sls_level2_spectra_header
    create_time          datetime,
    primary key (id)
 );
--- csst.msc_level2_catalog definition
 
+/*------------------------------------------------*/
 CREATE TABLE msc_level2_catalog (  
-  source_id integer PRIMARY KEY autoincrement,
-  obs_id varchar(12),
-  detector_no char(2),
   seq int(20),
   flux_aper_1 double,
   flux_aper_2 double,
@@ -890,5 +908,10 @@ CREATE TABLE msc_level2_catalog (
   disk_theta_world double,
   disk_thetaerr_world double,
   disk_theta_j2000 double,
-  obs_time datetime
+  level2_id integer,
+  NS8HIdx integer,
+  NS16HIdx integer,
+  NS32HIdx integer,
+  NS64HIdx integer,
+  create_time datetime
 ) ;
\ No newline at end of file

csst_dfs_api_local/ifs/ingest.py

@@ -6,6 +6,8 @@ import datetime
 import shutil
 
 from csst_dfs_api_local.common.db import DBClient
+from csst_dfs_commons.utils.fits import get_header_value
+
 log = logging.getLogger('csst-dfs-api-local')
 
 def ingest():
@@ -68,8 +70,9 @@ def ingest_one(file_path, db, copyfiles):
         (obs_id,exp_start_time,exp_time,module_id,obs_type,facility_status_id,module_status_id,qc0_status, prc_status,create_time))
         db.end()
     #level0
-    detector = header1["DETNAM"]
+    detector = get_header_value("DETNAM", header1, "-")
     filename = header["FILENAME"]
+    version = get_header_value("IMG_VER", header, "-")
     
     existed = db.exists(
             "select * from ifs_level0_data where filename=?",
@@ -93,9 +96,9 @@ def ingest_one(file_path, db, copyfiles):
     level0_id = f"{obs_id}{detector}"  
 
     c = db.execute("insert into ifs_level0_data \
-        (level0_id, obs_id, detector_no, object_name, obs_type, obs_time, exp_time,detector_status_id, filename, file_path,qc0_status, prc_status,create_time) \
-        values (?,?,?,?,?,?,?,?,?,?,?,?,?)",
-        (level0_id, obs_id, detector, object_name, obs_type, exp_start_time, exp_time, detector_status_id, filename, file_full_path, qc0_status, prc_status,create_time))
+        (level0_id, obs_id, detector_no, obs_type, obs_time, exp_time,detector_status_id, filename, file_path,qc0_status, prc_status,create_time) \
+        values (?,?,?,?,?,?,?,?,?,?,?,?)",
+        (level0_id, obs_id, detector, obs_type, exp_start_time, exp_time, detector_status_id, filename, file_full_path, qc0_status, prc_status,create_time))
     db.end()
     level0_id_id = db.last_row_id()
     #level0-header
@@ -103,9 +106,9 @@ def ingest_one(file_path, db, copyfiles):
     dec_obj = header["OBJ_DEC"]
     db.execute("delete from ifs_level0_header where id=?",(level0_id_id,))    
     db.execute("insert into ifs_level0_header \
-        (id, obs_time, exp_time, ra, `dec`, create_time) \
-        values (?,?,?,?,?,?)",
-        (level0_id_id, exp_start_time, exp_time, ra_obj, dec_obj, create_time))
+        (id, ra, `dec`, object_name, version) \
+        values (?,?,?,?,?)",
+        (level0_id_id, ra_obj, dec_obj, object_name, version))
     
     if copyfiles:
         #copy files

csst_dfs_api_local/ifs/level0.py

@@ -38,35 +38,46 @@ class Level0DataApi(object):
             detector_no = get_parameter(kwargs, "detector_no")
             obs_type = get_parameter(kwargs, "obs_type")
             object_name = get_parameter(kwargs, "object_name")
+            version = get_parameter(kwargs, "version")
             exp_time_start = get_parameter(kwargs, "obs_time", [None, None])[0]
             exp_time_end = get_parameter(kwargs, "obs_time", [None, None])[1]
             qc0_status = get_parameter(kwargs, "qc0_status")
             prc_status = get_parameter(kwargs, "prc_status")
             file_name = get_parameter(kwargs, "file_name")
             limit = get_parameter(kwargs, "limit", 0)
+            ra = get_parameter(kwargs, "ra", None)
+            dec = get_parameter(kwargs, "limit", None)
+            radius = get_parameter(kwargs, "limit", 0)
 
-            sql_count = "select count(*) as c from ifs_level0_data where 1=1"
-            sql_data = f"select * from ifs_level0_data where 1=1"
+            sql_count = 'select count(*) as c from ifs_level0_data d left join ifs_level0_header h on d.id=h.id where 1=1'
+            sql_data = 'select d.* from ifs_level0_data d left join ifs_level0_header h on d.id=h.id where 1=1'
 
             sql_condition = "" 
             if obs_id:
-                sql_condition = f"{sql_condition} and obs_id='{obs_id}'"              
+                sql_condition = f"{sql_condition} and d.obs_id='{obs_id}'"              
             if detector_no:
-                sql_condition = f"{sql_condition} and detector_no='{detector_no}'"
+                sql_condition = f"{sql_condition} and d.detector_no='{detector_no}'"
             if obs_type:
-                sql_condition = f"{sql_condition} and obs_type='{obs_type}'"
-            if object_name:
-                sql_condition = f"{sql_condition} and object_name='{object_name}'"                
+                sql_condition = f"{sql_condition} and d.obs_type='{obs_type}'"
             if exp_time_start:
-                sql_condition = f"{sql_condition} and obs_time >='{exp_time_start}'"
+                sql_condition = f"{sql_condition} and d.obs_time >='{exp_time_start}'"
             if exp_time_end:
-                sql_condition = f"{sql_condition} and obs_time <='{exp_time_end}'"
+                sql_condition = f"{sql_condition} and d.obs_time <='{exp_time_end}'"
             if qc0_status:
-                sql_condition = f"{sql_condition} and qc0_status={qc0_status}"
+                sql_condition = f"{sql_condition} and d.qc0_status={qc0_status}"
             if prc_status:
-                sql_condition = f"{sql_condition} and prc_status={prc_status}"   
+                sql_condition = f"{sql_condition} and d.prc_status={prc_status}"  
+            if object_name:
+                sql_condition = f"{sql_condition} and h.object_name='{object_name}'"
+            if version:
+                sql_condition = f"{sql_condition} and h.version='{version}'"  
+            if ra:
+                sql_condition = f"{sql_condition} and (h.ra <= {ra+radius} and h.ra >={ra-radius})"  
+            if dec:
+                sql_condition = f"{sql_condition} and (h.dec <= {dec+radius} and h.ra >={dec-radius})"
+
             if file_name:
-                sql_condition = f" and filename='{file_name}'"  
+                sql_condition = f" and d.filename='{file_name}'"  
 
             sql_count = f"{sql_count} {sql_condition}"
             sql_data = f"{sql_data} {sql_condition}"
@@ -204,7 +215,6 @@ class Level0DataApi(object):
             obs_id = get_parameter(kwargs, "obs_id"),
             detector_no = get_parameter(kwargs, "detector_no"),
             obs_type = get_parameter(kwargs, "obs_type"),
-            object_name = get_parameter(kwargs, "object_name"),
             obs_time = get_parameter(kwargs, "obs_time"),
             exp_time = get_parameter(kwargs, "exp_time"),
             detector_status_id = get_parameter(kwargs, "detector_status_id"),
@@ -222,9 +232,9 @@ class Level0DataApi(object):
                 return Result.error(message ='%s existed' %(rec.filename, ))
 
             self.db.execute(
-                'INSERT INTO ifs_level0_data (level0_id, obs_id, detector_no, object_name, obs_type, obs_time, exp_time,detector_status_id, filename, file_path,qc0_status, prc_status,create_time) \
+                'INSERT INTO ifs_level0_data (level0_id, obs_id, detector_no, obs_type, obs_time, exp_time,detector_status_id, filename, file_path,qc0_status, prc_status,create_time) \
                     VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?)',
-                (rec.level0_id, rec.obs_id, rec.detector_no, rec.object_name, rec.obs_type, rec.obs_time, rec.exp_time, rec.detector_status_id, rec.filename, rec.file_path,-1,-1,format_time_ms(time.time()))
+                (rec.level0_id, rec.obs_id, rec.detector_no, rec.obs_type, rec.obs_time, rec.exp_time, rec.detector_status_id, rec.filename, rec.file_path,-1,-1,format_time_ms(time.time()))
             )
             self.db.end()
             rec.id = self.db.last_row_id()

csst_dfs_api_local/mci/ingest.py

@@ -6,6 +6,8 @@ import datetime
 import shutil
 
 from csst_dfs_api_local.common.db import DBClient
+from csst_dfs_commons.utils.fits import get_header_value
+
 log = logging.getLogger('csst-dfs-api-local')
 
 def ingest():
@@ -51,6 +53,8 @@ def ingest_one(file_path, db, copyfiles):
     
     module_id = header["INSTRUME"]
     obs_type = header["OBSTYPE"]
+    object_name = header["OBJECT"]
+
     qc0_status = -1
     prc_status = -1
     time_now = datetime.datetime.now()
@@ -67,9 +71,9 @@ def ingest_one(file_path, db, copyfiles):
         (obs_id,exp_start_time,exp_time,module_id,obs_type,facility_status_id,module_status_id,qc0_status, prc_status,create_time))
         db.end()
     #level0
-    detector = header1["DETNAM"]
+    detector = get_header_value("DETNAM", header1, "-")
     filename = header["FILENAME"]
-    
+    version = get_header_value("IMG_VER", header, "-")
     existed = db.exists(
             "select * from mci_level0_data where filename=?",
             (filename,)
@@ -102,9 +106,9 @@ def ingest_one(file_path, db, copyfiles):
     dec_obj = header["OBJ_DEC"]
     db.execute("delete from mci_level0_header where id=?",(level0_id_id,))    
     db.execute("insert into mci_level0_header \
-        (id, obs_time, exp_time, ra, `dec`, create_time) \
-        values (?,?,?,?,?,?)",
-        (level0_id_id, exp_start_time, exp_time, ra_obj, dec_obj, create_time))
+        (id, ra, `dec`, object_name, version) \
+        values (?,?,?,?,?)",
+        (level0_id_id, ra_obj, dec_obj, object_name, version))
     
     if copyfiles:
         #copy files

csst_dfs_api_local/msc/ingest.py

@@ -6,6 +6,8 @@ import datetime
 import shutil
 
 from csst_dfs_api_local.common.db import DBClient
+from csst_dfs_commons.utils.fits import get_header_value
+
 log = logging.getLogger('csst-dfs-api-local')
 
 def ingest():