diff --git a/.hypothesis/unicode_data/11.0.0/charmap.json.gz b/.hypothesis/unicode_data/11.0.0/charmap.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..c9dcf3fbbf5ffc938934091fab8df93f59b37283 Binary files /dev/null and b/.hypothesis/unicode_data/11.0.0/charmap.json.gz differ diff --git a/csst_dfs_api_local/common/db.sql b/csst_dfs_api_local/common/db.sql index b49d6d7974afa5f2847ac4e21648258444d149ef..477ae2ccf4ee9f8c884bba0b13c0b54a93152464 100644 --- a/csst_dfs_api_local/common/db.sql +++ b/csst_dfs_api_local/common/db.sql @@ -29,7 +29,64 @@ drop table if exists t_level0_prc; drop table if exists t_module_status; drop table if exists t_observation; - +/*==============================================================*/ +/* Table: sls_level1_data */ +/*==============================================================*/ +create table sls_level1_data +( + id integer PRIMARY KEY autoincrement, + level0_id varchar(20) not null, + data_type varchar(64) not null, + prc_params varchar(1024), + filename varchar(128), + file_path varchar(256), + prc_status tinyint(1), + prc_time datetime, + qc1_status tinyint(1), + qc1_time datetime, + create_time datetime, + pipeline_id varchar(60) +); +/*==============================================================*/ +/* Table: sls_level1_ref */ +/*==============================================================*/ +create table sls_level1_ref ( + level1_id int(20) not null, + ref_type varchar(64) not null, + cal_id int(20) not null, + primary key (level1_id, ref_type) +); +/*==============================================================*/ +/* Table: sls_level1_header */ +/*==============================================================*/ +create table sls_level1_header +( + id int(20) not null, + obs_time datetime, + exp_time float, + ra float, + "dec" float, + create_time datetime, + primary key (id) +); +/*==============================================================*/ +/* Table: sls_level2_spectra */ +/*==============================================================*/ +create table sls_level2_spectra +( + id integer PRIMARY KEY autoincrement, + spectra_id varchar(40), + level1_id int(20) not null, + region varchar(128), + filename varchar(128), + file_path varchar(256), + prc_status tinyint(1), + prc_time datetime, + qc1_status tinyint(1), + qc1_time datetime, + create_time datetime, + pipeline_id varchar(60) +); /*==============================================================*/ /* Table: ifs_level1_data */ /*==============================================================*/ @@ -40,12 +97,6 @@ create table ifs_level1_data data_type varchar(64) not null, cor_sci_id int(20), prc_params varchar(1024), - flat_id int(20), - dark_id int(20), - bias_id int(20), - lamp_id int(20), - arc_id int(20), - sky_id int(20), filename varchar(128), file_path varchar(256), prc_status tinyint(1), @@ -55,7 +106,15 @@ create table ifs_level1_data create_time datetime, pipeline_id varchar(60) ); - +/*==============================================================*/ +/* Table: ifs_level1_ref */ +/*==============================================================*/ +create table ifs_level1_ref ( + level1_id int(20) not null, + ref_type varchar(64) not null, + cal_id int(20) not null, + primary key (level1_id, ref_type) +); /*==============================================================*/ /* Table: ifs_level1_header */ /*==============================================================*/ @@ -80,9 +139,6 @@ create table msc_level1_data data_type varchar(64) not null, cor_sci_id int(20), prc_params varchar(1024), - flat_id int(20), - dark_id int(20), - bias_id int(20), filename varchar(128), file_path varchar(256), prc_status tinyint(1), @@ -92,7 +148,15 @@ create table msc_level1_data create_time datetime, pipeline_id varchar(60) ); - +/*==============================================================*/ +/* Table: msc_level1_ref */ +/*==============================================================*/ +create table msc_level1_ref ( + level1_id int(20) not null, + ref_type varchar(64) not null, + cal_id int(20) not null, + primary key (level1_id, ref_type) +); /*==============================================================*/ /* Table: msc_level1_header */ /*==============================================================*/ @@ -250,7 +314,20 @@ create table t_level0_prc prc_time datetime, result_file_path varchar(256) ); - +/*==============================================================*/ +/* Table: t_level1_prc */ +/*==============================================================*/ +create table t_level1_prc +( + id integer PRIMARY KEY autoincrement, + level1_id int(20) not null, + pipeline_id varchar(64) not null, + prc_module varchar(32) not null, + params_file_path varchar(256), + prc_status int(2), + prc_time datetime, + result_file_path varchar(256) +); /*==============================================================*/ /* Table: t_module_status */ /*==============================================================*/ diff --git a/csst_dfs_api_local/facility/level1prc.py b/csst_dfs_api_local/facility/level1prc.py new file mode 100644 index 0000000000000000000000000000000000000000..851e5a569f3e3d3e92e2538cb1867a868227e1d0 --- /dev/null +++ b/csst_dfs_api_local/facility/level1prc.py @@ -0,0 +1,123 @@ +import os +import logging +import time, datetime +import shutil + +from ..common.db import DBClient +from ..common.utils import * +from csst_dfs_commons.models import Result +from csst_dfs_commons.models.facility import Level1PrcRecord +from csst_dfs_commons.models.common import from_dict_list + +log = logging.getLogger('csst') + +class Level1PrcApi(object): + def __init__(self): + self.root_dir = os.getenv("CSST_LOCAL_FILE_ROOT", "/opt/temp/csst") + self.db = DBClient() + + def find(self, **kwargs): + ''' retrieve level1 procedure records from database + + parameter kwargs: + level1_id: [int] + pipeline_id: [str] + prc_module: [str] + prc_status : [int] + + return: csst_dfs_common.models.Result + ''' + try: + level1_id = get_parameter(kwargs, "level1_id", 0) + pipeline_id = get_parameter(kwargs, "pipeline_id") + prc_module = get_parameter(kwargs, "prc_module") + prc_status = get_parameter(kwargs, "prc_status") + + sql_data = f"select * from t_level1_prc" + + sql_condition = f"where level1_id={level1_id}" + if pipeline_id: + sql_condition = sql_condition + " and pipeline_id='" + pipeline_id + "'" + if prc_module: + sql_condition = sql_condition + " and prc_module ='" + prc_module + "'" + if prc_status: + sql_condition = f"{sql_condition} and prc_status={prc_status}" + + sql_data = f"{sql_data} {sql_condition}" + + _, records = self.db.select_many(sql_data) + return Result.ok_data(data=from_dict_list(Level1PrcRecord, records)).append("totalCount", len(records)) + + except Exception as e: + return Result.error(message=str(e)) + + def update_proc_status(self, **kwargs): + ''' update the status of reduction + + parameter kwargs: + id : [int], + status : [int] + + return csst_dfs_common.models.Result + ''' + id = get_parameter(kwargs, "id") + status = get_parameter(kwargs, "status") + + try: + existed = self.db.exists( + "select * from t_level1_prc where id=?", + (id,) + ) + if not existed: + log.warning('%s not found' %(id, )) + return Result.error(message ='%s not found' %(id, )) + self.db.execute( + 'update t_level1_prc set prc_status=?, prc_time=? where id=?', + (status, format_time_ms(time.time()), id) + ) + self.db.end() + return Result.ok_data() + + except Exception as e: + log.error(e) + return Result.error(message=str(e)) + + def write(self, **kwargs): + ''' insert a level1 procedure record into database + + parameter kwargs: + level1_id : [int] + pipeline_id : [str] + prc_module : [str] + params_file_path : [str] + prc_status : [int] + prc_time : [str] + result_file_path : [str] + return csst_dfs_common.models.Result + ''' + + rec = Level1PrcRecord( + id = 0, + level1_id = get_parameter(kwargs, "level1_id", 0), + pipeline_id = get_parameter(kwargs, "pipeline_id"), + prc_module = get_parameter(kwargs, "prc_module"), + params_file_path = get_parameter(kwargs, "params_file_path"), + prc_status = get_parameter(kwargs, "prc_status", -1), + prc_time = get_parameter(kwargs, "prc_time"), + result_file_path = get_parameter(kwargs, "result_file_path") + ) + try: + self.db.execute( + 'INSERT INTO t_level1_prc (level1_id,pipeline_id,prc_module, params_file_path, prc_status,prc_time,result_file_path) \ + VALUES(?,?,?,?,?,?,?)', + (rec.level1_id, rec.pipeline_id, rec.prc_module, rec.params_file_path, rec.prc_status, rec.prc_time, rec.result_file_path) + ) + self.db.end() + rec.id = self.db.last_row_id() + + return Result.ok_data(data=rec) + + except Exception as e: + log.error(e) + return Result.error(message=str(e)) + diff --git a/csst_dfs_api_local/ifs/level1.py b/csst_dfs_api_local/ifs/level1.py index d0249507a108be9c9ed4726ca2f94e4ffcafb277..812dbd76283055fcd0ae4bf5445d0039b635b221 100644 --- a/csst_dfs_api_local/ifs/level1.py +++ b/csst_dfs_api_local/ifs/level1.py @@ -23,7 +23,6 @@ class Level1DataApi(object): parameter kwargs: level0_id: [str] data_type: [str] - obs_type: [str] create_time : (start, end), qc1_status : [int], prc_status : [int], @@ -161,17 +160,12 @@ class Level1DataApi(object): data_type : [str] cor_sci_id : [int] prc_params : [str] - flat_id : [int] - dark_id : [int] - bias_id : [int] - lamp_id : [int] - arc_id : [int] - sky_id : [int] filename : [str] file_path : [str] prc_status : [int] prc_time : [str] pipeline_id : [str] + refs : [dict] return csst_dfs_common.models.Result ''' @@ -182,17 +176,12 @@ class Level1DataApi(object): data_type = get_parameter(kwargs, "data_type"), cor_sci_id = get_parameter(kwargs, "cor_sci_id"), prc_params = get_parameter(kwargs, "prc_params"), - flat_id = get_parameter(kwargs, "flat_id"), - dark_id = get_parameter(kwargs, "dark_id"), - bias_id = get_parameter(kwargs, "bias_id"), - lamp_id = get_parameter(kwargs, "lamp_id"), - arc_id = get_parameter(kwargs, "arc_id"), - sky_id = get_parameter(kwargs, "sky_id"), filename = get_parameter(kwargs, "filename"), file_path = get_parameter(kwargs, "file_path"), prc_status = get_parameter(kwargs, "prc_status", -1), prc_time = get_parameter(kwargs, "prc_time", format_datetime(datetime.now())), - pipeline_id = get_parameter(kwargs, "pipeline_id") + pipeline_id = get_parameter(kwargs, "pipeline_id"), + refs = get_parameter(kwargs, "refs", {}) ) existed = self.db.exists( "select * from ifs_level1_data where filename=?", @@ -202,14 +191,22 @@ class Level1DataApi(object): log.error(f'{rec.filename} has already been existed') return Result.error(message=f'{rec.filename} has already been existed') + now_str = format_time_ms(time.time()) self.db.execute( - 'INSERT INTO ifs_level1_data (level0_id,data_type,cor_sci_id,prc_params,flat_id,dark_id,bias_id,lamp_id,arc_id,sky_id,filename,file_path,qc1_status,prc_status,prc_time, create_time,pipeline_id) \ - VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)', - (rec.level0_id, rec.data_type, rec.cor_sci_id, rec.prc_params, rec.flat_id, rec.dark_id, rec.bias_id, rec.lamp_id, rec.arc_id, rec.sky_id, rec.filename, rec.file_path, -1, rec.prc_status,rec.prc_time, format_time_ms(time.time()),rec.pipeline_id,) + 'INSERT INTO ifs_level1_data (level0_id,data_type,cor_sci_id,prc_params,filename,file_path,qc1_status,prc_status,prc_time, create_time,pipeline_id) \ + VALUES(?,?,?,?,?,?,?,?,?,?,?)', + (rec.level0_id, rec.data_type, rec.cor_sci_id, rec.prc_params, rec.filename, rec.file_path, -1, rec.prc_status,rec.prc_time, now_str, rec.pipeline_id,) ) self.db.end() rec.id = self.db.last_row_id() + if rec.refs.items(): + sql_refs = "insert into ifs_level1_ref (level1_id,ref_type,cal_id) values " + values = ["(%s,'%s',%s)"%(rec.id,k,v) for k,v in rec.refs.items()] + _ = self.db.execute(sql_refs + ",".join(values)) + self.db.end() + + rec.create_time = now_str return Result.ok_data(data=rec) except Exception as e: log.error(e) diff --git a/csst_dfs_api_local/msc/level1.py b/csst_dfs_api_local/msc/level1.py index 160cb71ba9f53900dc456842ebcb14b52de3c64b..9e1a9130d5b9d0e8b2268e5856e2eafdcb92f8d2 100644 --- a/csst_dfs_api_local/msc/level1.py +++ b/csst_dfs_api_local/msc/level1.py @@ -23,7 +23,6 @@ class Level1DataApi(object): parameter kwargs: level0_id: [str] data_type: [str] - obs_type: [str] create_time : (start, end), qc1_status : [int], prc_status : [int], @@ -74,28 +73,6 @@ class Level1DataApi(object): except Exception as e: return Result.error(message=str(e)) - - def get(self, **kwargs): - ''' - parameter kwargs: - id = [int], - level0_id = [str] - - return dict or None - ''' - try: - fits_id = get_parameter(kwargs, "id", -1) - r = self.db.select_one( - "select * from msc_level1_data where id=?", (fits_id,)) - - if r: - return Result.ok_data(data=Level1Record().from_dict(r)) - else: - return Result.error(message=f"id:{fits_id} not found") - except Exception as e: - log.error(e) - return Result.error(message=str(e)) - def get(self, **kwargs): ''' parameter kwargs: @@ -180,18 +157,13 @@ class Level1DataApi(object): data_type : [str] cor_sci_id : [int] prc_params : [str] - flat_id : [int] - dark_id : [int] - bias_id : [int] - lamp_id : [int] - arc_id : [int] - sky_id : [int] filename : [str] file_path : [str] prc_status : [int] prc_time : [str] pipeline_id : [str] - + refs: [dict] + return csst_dfs_common.models.Result ''' try: @@ -201,14 +173,12 @@ class Level1DataApi(object): data_type = get_parameter(kwargs, "data_type"), cor_sci_id = get_parameter(kwargs, "cor_sci_id"), prc_params = get_parameter(kwargs, "prc_params"), - flat_id = get_parameter(kwargs, "flat_id"), - dark_id = get_parameter(kwargs, "dark_id"), - bias_id = get_parameter(kwargs, "bias_id"), filename = get_parameter(kwargs, "filename"), file_path = get_parameter(kwargs, "file_path"), prc_status = get_parameter(kwargs, "prc_status", -1), prc_time = get_parameter(kwargs, "prc_time", format_datetime(datetime.now())), - pipeline_id = get_parameter(kwargs, "pipeline_id") + pipeline_id = get_parameter(kwargs, "pipeline_id"), + refs = get_parameter(kwargs, "refs", {}) ) existed = self.db.exists( "select * from msc_level1_data where filename=?", @@ -219,13 +189,19 @@ class Level1DataApi(object): return Result.error(message=f'{rec.filename} has already been existed') self.db.execute( - 'INSERT INTO msc_level1_data (level0_id,data_type,cor_sci_id,prc_params,flat_id,dark_id,bias_id,filename,file_path,qc1_status,prc_status,prc_time,create_time,pipeline_id) \ - VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?)', - (rec.level0_id, rec.data_type, rec.cor_sci_id, rec.prc_params, rec.flat_id, rec.dark_id, rec.bias_id, rec.filename, rec.file_path, -1, rec.prc_status, rec.prc_time, format_time_ms(time.time()),rec.pipeline_id,) + 'INSERT INTO msc_level1_data (level0_id,data_type,cor_sci_id,prc_params,filename,file_path,qc1_status,prc_status,prc_time,create_time,pipeline_id) \ + VALUES(?,?,?,?,?,?,?,?,?,?,?)', + (rec.level0_id, rec.data_type, rec.cor_sci_id, rec.prc_params, rec.filename, rec.file_path, -1, rec.prc_status, rec.prc_time, format_time_ms(time.time()),rec.pipeline_id,) ) self.db.end() rec.id = self.db.last_row_id() + if rec.refs.items(): + sql_refs = "insert into msc_level1_ref (level1_id,ref_type,cal_id) values " + values = ["(%s,'%s',%s)"%(rec.id,k,v) for k,v in rec.refs.items()] + _ = self.db.execute(sql_refs + ",".join(values)) + + self.db.end() return Result.ok_data(data=rec) except Exception as e: log.error(e) diff --git a/csst_dfs_api_local/sls/__init__.py b/csst_dfs_api_local/sls/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..819de80eeba5ef22911a8443de66f86fe63b39d8 --- /dev/null +++ b/csst_dfs_api_local/sls/__init__.py @@ -0,0 +1,2 @@ +from .level1 import Level1DataApi +from .level2spectra import Level2SpectraApi \ No newline at end of file diff --git a/csst_dfs_api_local/sls/level1.py b/csst_dfs_api_local/sls/level1.py new file mode 100644 index 0000000000000000000000000000000000000000..60d8e8d73209a6635e47042f8956f218f819c178 --- /dev/null +++ b/csst_dfs_api_local/sls/level1.py @@ -0,0 +1,205 @@ +import os +import logging +import time, datetime +import shutil + +from ..common.db import DBClient +from ..common.utils import * +from csst_dfs_commons.models import Result +from csst_dfs_commons.models.sls import Level1Record +from csst_dfs_commons.models.common import from_dict_list + +log = logging.getLogger('csst') + +class Level1DataApi(object): + def __init__(self, sub_system = "sls"): + self.sub_system = sub_system + self.root_dir = os.getenv("CSST_LOCAL_FILE_ROOT", "/opt/temp/csst") + self.db = DBClient() + + def find(self, **kwargs): + ''' retrieve level1 records from database + + parameter kwargs: + level0_id: [str] + data_type: [str] + create_time : (start, end), + qc1_status : [int], + prc_status : [int], + filename: [str] + limit: limits returns the number of records,default 0:no-limit + + return: csst_dfs_common.models.Result + ''' + try: + level0_id = get_parameter(kwargs, "level0_id") + data_type = get_parameter(kwargs, "data_type") + create_time_start = get_parameter(kwargs, "create_time", [None, None])[0] + create_time_end = get_parameter(kwargs, "create_time", [None, None])[1] + qc1_status = get_parameter(kwargs, "qc1_status") + prc_status = get_parameter(kwargs, "prc_status") + filename = get_parameter(kwargs, "filename") + limit = get_parameter(kwargs, "limit", 0) + + sql_count = "select count(*) as c from sls_level1_data where 1=1" + sql_data = f"select * from sls_level1_data where 1=1" + + sql_condition = "" + if level0_id: + sql_condition = f"{sql_condition} and level0_id='{level0_id}'" + if data_type: + sql_condition = f"{sql_condition} and data_type='{data_type}'" + if create_time_start: + sql_condition = f"{sql_condition} and create_time >='{create_time_start}'" + if create_time_end: + sql_condition = f"{sql_condition} and create_time <='{create_time_end}'" + if qc1_status: + sql_condition = f"{sql_condition} and qc1_status={qc1_status}" + if prc_status: + sql_condition = f"{sql_condition} and prc_status={prc_status}" + if filename: + sql_condition = f" and filename='{filename}'" + + sql_count = f"{sql_count} {sql_condition}" + sql_data = f"{sql_data} {sql_condition}" + + if limit > 0: + sql_data = f"{sql_data} limit {limit}" + + totalCount = self.db.select_one(sql_count) + _, recs = self.db.select_many(sql_data) + return Result.ok_data(data=from_dict_list(Level1Record, recs)).append("totalCount", totalCount['c']) + + except Exception as e: + return Result.error(message=str(e)) + + def get(self, **kwargs): + ''' + parameter kwargs: + id = [int] + return csst_dfs_common.models.Result + ''' + try: + id = get_parameter(kwargs, "id", -1) + r = self.db.select_one( + "select * from sls_level1_data where id=?", (id,)) + if r: + return Result.ok_data(data=Level1Record().from_dict(r)) + else: + return Result.error(message=f"id:{id} not found") + except Exception as e: + log.error(e) + return Result.error(message=str(e)) + + def update_proc_status(self, **kwargs): + ''' update the status of reduction + + parameter kwargs: + id : [int], + status : [int] + + return csst_dfs_common.models.Result + ''' + fits_id = get_parameter(kwargs, "id") + status = get_parameter(kwargs, "status") + try: + existed = self.db.exists( + "select * from sls_level1_data where id=?", + (fits_id,) + ) + if not existed: + log.warning('%s not found' %(fits_id, )) + return Result.error(message ='%s not found' %(fits_id, )) + self.db.execute( + 'update sls_level1_data set prc_status=?, prc_time=? where id=?', + (status, format_time_ms(time.time()), fits_id) + ) + self.db.end() + return Result.ok_data() + + except Exception as e: + log.error(e) + return Result.error(message=str(e)) + + def update_qc1_status(self, **kwargs): + ''' update the status of QC1 + + parameter kwargs: + id : [int], + status : [int] + ''' + fits_id = get_parameter(kwargs, "id") + status = get_parameter(kwargs, "status") + try: + existed = self.db.exists( + "select * from sls_level1_data where id=?", + (fits_id,) + ) + if not existed: + log.warning('%s not found' %(fits_id, )) + return Result.error(message ='%s not found' %(fits_id, )) + self.db.execute( + 'update sls_level1_data set qc1_status=?, qc1_time=? where id=?', + (status, format_time_ms(time.time()), fits_id) + ) + self.db.end() + return Result.ok_data() + + except Exception as e: + log.error(e) + return Result.error(message=str(e)) + + def write(self, **kwargs): + ''' insert a level1 record into database + + parameter kwargs: + level0_id : [str] + data_type : [str] + prc_params : [str] + filename : [str] + file_path : [str] + prc_status : [int] + prc_time : [str] + pipeline_id : [str] + refs: [dict] + return csst_dfs_common.models.Result + ''' + try: + rec = Level1Record( + id = 0, + level0_id = get_parameter(kwargs, "level0_id"), + data_type = get_parameter(kwargs, "data_type"), + prc_params = get_parameter(kwargs, "prc_params"), + filename = get_parameter(kwargs, "filename"), + file_path = get_parameter(kwargs, "file_path"), + prc_status = get_parameter(kwargs, "prc_status", -1), + prc_time = get_parameter(kwargs, "prc_time", format_datetime(datetime.now())), + pipeline_id = get_parameter(kwargs, "pipeline_id"), + refs = get_parameter(kwargs, "refs", {}) + ) + existed = self.db.exists( + "select * from sls_level1_data where filename=?", + (rec.filename,) + ) + if existed: + log.error(f'{rec.filename} has already been existed') + return Result.error(message=f'{rec.filename} has already been existed') + + self.db.execute( + 'INSERT INTO sls_level1_data (level0_id,data_type,prc_params,filename,file_path,qc1_status,prc_status,prc_time,create_time,pipeline_id) \ + VALUES(?,?,?,?,?,?,?,?,?,?)', + (rec.level0_id, rec.data_type, rec.prc_params, rec.filename, rec.file_path, -1, rec.prc_status, rec.prc_time, format_time_ms(time.time()),rec.pipeline_id,) + ) + self.db.end() + rec.id = self.db.last_row_id() + + if rec.refs.items(): + sql_refs = "insert into sls_level1_ref (level1_id,ref_type,cal_id) values " + values = ["(%s,'%s',%s)"%(rec.id,k,v) for k,v in rec.refs.items()] + _ = self.db.execute(sql_refs + ",".join(values)) + + self.db.end() + return Result.ok_data(data=rec) + except Exception as e: + log.error(e) + return Result.error(message=str(e)) \ No newline at end of file diff --git a/csst_dfs_api_local/sls/level2spectra.py b/csst_dfs_api_local/sls/level2spectra.py new file mode 100644 index 0000000000000000000000000000000000000000..d9cf2be26e1e1974f14f67f1abf4b6c69f1b0942 --- /dev/null +++ b/csst_dfs_api_local/sls/level2spectra.py @@ -0,0 +1,198 @@ +import os +import logging +import time, datetime +import shutil + +from ..common.db import DBClient +from ..common.utils import * +from csst_dfs_commons.models import Result +from csst_dfs_commons.models.sls import Level2Spectra +from csst_dfs_commons.models.common import from_dict_list + +log = logging.getLogger('csst') + +class Level2SpectraApi(object): + def __init__(self, sub_system = "msc"): + self.sub_system = sub_system + self.root_dir = os.getenv("CSST_LOCAL_FILE_ROOT", "/opt/temp/csst") + self.db = DBClient() + + def find(self, **kwargs): + ''' retrieve records from database + + parameter kwargs: + level1_id: [int] + spectra_id: [str] + create_time : (start, end), + qc1_status : [int], + prc_status : [int], + filename: [str] + limit: limits returns the number of records,default 0:no-limit + + return: csst_dfs_common.models.Result + ''' + try: + level1_id = get_parameter(kwargs, "level1_id", 0) + spectra_id = get_parameter(kwargs, "spectra_id") + create_time_start = get_parameter(kwargs, "create_time", [None, None])[0] + create_time_end = get_parameter(kwargs, "create_time", [None, None])[1] + qc1_status = get_parameter(kwargs, "qc1_status") + prc_status = get_parameter(kwargs, "prc_status") + filename = get_parameter(kwargs, "filename") + limit = get_parameter(kwargs, "limit", 0) + + sql_count = "select count(*) as c from sls_level2_spectra where 1=1" + sql_data = f"select * from sls_level2_spectra where 1=1" + + sql_condition = "" + if level1_id > 0: + sql_condition = f"{sql_condition} and level1_id={level1_id}" + if spectra_id: + sql_condition = f"{sql_condition} and spectra_id='{spectra_id}'" + if create_time_start: + sql_condition = f"{sql_condition} and create_time >='{create_time_start}'" + if create_time_end: + sql_condition = f"{sql_condition} and create_time <='{create_time_end}'" + if qc1_status: + sql_condition = f"{sql_condition} and qc1_status={qc1_status}" + if prc_status: + sql_condition = f"{sql_condition} and prc_status={prc_status}" + if filename: + sql_condition = f" and filename='{filename}'" + + sql_count = f"{sql_count} {sql_condition}" + sql_data = f"{sql_data} {sql_condition}" + + if limit > 0: + sql_data = f"{sql_data} limit {limit}" + + totalCount = self.db.select_one(sql_count) + _, recs = self.db.select_many(sql_data) + return Result.ok_data(data=from_dict_list(Level2Spectra, recs)).append("totalCount", totalCount['c']) + + except Exception as e: + return Result.error(message=str(e)) + + def get(self, **kwargs): + ''' + parameter kwargs: + id = [int] + return csst_dfs_common.models.Result + ''' + try: + id = get_parameter(kwargs, "id", -1) + r = self.db.select_one( + "select * from sls_level2_spectra where id=?", (id,)) + if r: + return Result.ok_data(data=Level2Spectra().from_dict(r)) + else: + return Result.error(message=f"id:{id} not found") + except Exception as e: + log.error(e) + return Result.error(message=str(e)) + + def update_proc_status(self, **kwargs): + ''' update the status of reduction + + parameter kwargs: + id : [int], + status : [int] + + return csst_dfs_common.models.Result + ''' + fits_id = get_parameter(kwargs, "id") + status = get_parameter(kwargs, "status") + try: + existed = self.db.exists( + "select * from sls_level2_spectra where id=?", + (fits_id,) + ) + if not existed: + log.warning('%s not found' %(fits_id, )) + return Result.error(message ='%s not found' %(fits_id, )) + self.db.execute( + 'update sls_level2_spectra set prc_status=?, prc_time=? where id=?', + (status, format_time_ms(time.time()), fits_id) + ) + self.db.end() + return Result.ok_data() + + except Exception as e: + log.error(e) + return Result.error(message=str(e)) + + def update_qc1_status(self, **kwargs): + ''' update the status of QC1 + + parameter kwargs: + id : [int], + status : [int] + ''' + fits_id = get_parameter(kwargs, "id") + status = get_parameter(kwargs, "status") + try: + existed = self.db.exists( + "select * from sls_level2_spectra where id=?", + (fits_id,) + ) + if not existed: + log.warning('%s not found' %(fits_id, )) + return Result.error(message ='%s not found' %(fits_id, )) + self.db.execute( + 'update sls_level2_spectra set qc1_status=?, qc1_time=? where id=?', + (status, format_time_ms(time.time()), fits_id) + ) + self.db.end() + return Result.ok_data() + + except Exception as e: + log.error(e) + return Result.error(message=str(e)) + + def write(self, **kwargs): + ''' insert a level1 record into database + + parameter kwargs: + level1_id: [int] + spectra_id : [str] + region : [str] + filename : [str] + file_path : [str] + prc_status : [int] + prc_time : [str] + pipeline_id : [str] + + return csst_dfs_common.models.Result + ''' + try: + rec = Level2Spectra( + id = 0, + level1_id = get_parameter(kwargs, "level1_id"), + spectra_id = get_parameter(kwargs, "spectra_id"), + region = get_parameter(kwargs, "region"), + filename = get_parameter(kwargs, "filename"), + file_path = get_parameter(kwargs, "file_path"), + prc_status = get_parameter(kwargs, "prc_status", -1), + prc_time = get_parameter(kwargs, "prc_time", format_datetime(datetime.now())), + pipeline_id = get_parameter(kwargs, "pipeline_id") + ) + existed = self.db.exists( + "select * from sls_level2_spectra where filename=?", + (rec.filename,) + ) + if existed: + log.error(f'{rec.filename} has already been existed') + return Result.error(message=f'{rec.filename} has already been existed') + + self.db.execute( + 'INSERT INTO sls_level2_spectra (level1_id,spectra_id,region,filename,file_path,qc1_status,prc_status,prc_time,create_time,pipeline_id) \ + VALUES(?,?,?,?,?,?,?,?,?,?)', + (rec.level1_id, rec.spectra_id, rec.region, rec.filename, rec.file_path, -1, rec.prc_status, rec.prc_time, format_time_ms(time.time()),rec.pipeline_id,) + ) + self.db.end() + rec.id = self.db.last_row_id() + + return Result.ok_data(data=rec) + except Exception as e: + log.error(e) + return Result.error(message=str(e)) \ No newline at end of file diff --git a/tests/test_facility_level1_prc.py b/tests/test_facility_level1_prc.py new file mode 100644 index 0000000000000000000000000000000000000000..299497d66d06a2177d7ba05bf1927ac02edcb304 --- /dev/null +++ b/tests/test_facility_level1_prc.py @@ -0,0 +1,28 @@ +import os +import unittest +from astropy.io import fits + +from csst_dfs_api_local.facility.level1prc import Level1PrcApi + +class Level1PrcTestCase(unittest.TestCase): + + def setUp(self): + self.api = Level1PrcApi() + + def test_find(self): + recs = self.api.find(level1_id=1) + print('find:', recs) + + def test_update_proc_status(self): + rec = self.api.update_proc_status(id = 1, status = 4) + print('update_proc_status:', rec) + + def test_write(self): + rec = self.api.write(level1_id=1, + pipeline_id = "P1", + prc_module = "QC0", + params_file_path = "/opt/dddasd.params", + prc_status = 3, + prc_time = '2021-06-04 11:12:13', + result_file_path = "/opt/dddasd.header") + print('write:', rec) \ No newline at end of file diff --git a/tests/test_ifs_level1.py b/tests/test_ifs_level1.py index 2ee158f3bd30e9d0598fd8f812825813eb64747c..98ec0f3ebc5e6f25d5403503ea17e298a6221429 100644 --- a/tests/test_ifs_level1.py +++ b/tests/test_ifs_level1.py @@ -32,15 +32,10 @@ class IFSResult1TestCase(unittest.TestCase): data_type = "sci", cor_sci_id = 2, prc_params = "/opt/dddasd.params", - flat_id = 1, - dark_id = 2, - bias_id = 3, - lamp_id = 4, - arc_id = 5, - sky_id = 6, prc_status = 3, prc_time = '2021-06-05 11:12:13', - filename = "dddasd23", + filename = "dddasd223234.fits", file_path = "/opt/dddasd23.fits", - pipeline_id = "P2") + pipeline_id = "P2", + refs = {'dark': 1, 'bias': 2, 'flat': 3 }) print('write:', rec) \ No newline at end of file diff --git a/tests/test_msc_level1.py b/tests/test_msc_level1.py index 6667860534aa79f945dffd29649d2ad4520ccd8d..9ba510fd37b36970aa9579f5ee918c7d29908a2a 100644 --- a/tests/test_msc_level1.py +++ b/tests/test_msc_level1.py @@ -31,12 +31,10 @@ class IFSResult1TestCase(unittest.TestCase): data_type = "sci", cor_sci_id = 2, prc_params = "/opt/dddasd.params", - flat_id = 1, - dark_id = 2, - bias_id = 3, prc_status = 3, prc_time = '2021-06-05 11:12:13', filename = "dddasd", file_path = "/opt/dddasd.fits", - pipeline_id = "P2") + pipeline_id = "P2", + refs = {'dark': 1, 'bias': 2, 'flat': 3 }) print('write:', rec) \ No newline at end of file diff --git a/tests/test_sls_level1.py b/tests/test_sls_level1.py new file mode 100644 index 0000000000000000000000000000000000000000..cdaad8819f9711cd41f011163d0fbc5f008e53e1 --- /dev/null +++ b/tests/test_sls_level1.py @@ -0,0 +1,40 @@ +import unittest + +from csst_dfs_api_local.sls import Level1DataApi + +class SLSResult1TestCase(unittest.TestCase): + + def setUp(self): + self.api = Level1DataApi() + + def test_find(self): + recs = self.api.find( + level0_id='0000223', + create_time = ("2021-06-01 11:12:13","2021-06-08 11:12:13") + ) + print('find:', recs) + + def test_get(self): + rec = self.api.get(id = 1) + print('get:', rec) + + def test_update_proc_status(self): + rec = self.api.update_proc_status(id = 1, status = 4) + print('update_proc_status:', rec) + + def test_update_qc1_status(self): + rec = self.api.update_qc1_status(id = 1, status = 7) + print('update_qc1_status:', rec) + + def test_write(self): + rec = self.api.write( + level0_id='0000223', + data_type = "sci", + prc_params = "/opt/dddasd.params", + prc_status = 3, + prc_time = '2021-06-05 11:12:13', + filename = "dddasd223234.fits", + file_path = "/opt/dddasd23.fits", + pipeline_id = "P2", + refs = {'dark': 1, 'bias': 2, 'flat': 3 }) + print('write:', rec) \ No newline at end of file diff --git a/tests/test_sls_level2_spectra.py b/tests/test_sls_level2_spectra.py new file mode 100644 index 0000000000000000000000000000000000000000..47425a6c81cbab94259c9ed4947cdc0cce90f7c0 --- /dev/null +++ b/tests/test_sls_level2_spectra.py @@ -0,0 +1,39 @@ +import unittest + +from csst_dfs_api_local.sls import Level2SpectraApi + +class SLSLevel2SpectraTestCase(unittest.TestCase): + + def setUp(self): + self.api = Level2SpectraApi() + + def test_find(self): + recs = self.api.find( + level1_id=1, + create_time = ("2021-06-01 11:12:13","2021-06-08 11:12:13") + ) + print('find:', recs) + + def test_get(self): + rec = self.api.get(id = 1) + print('get:', rec) + + def test_update_proc_status(self): + rec = self.api.update_proc_status(id = 1, status = 4) + print('update_proc_status:', rec) + + def test_update_qc1_status(self): + rec = self.api.update_qc1_status(id = 1, status = 7) + print('update_qc1_status:', rec) + + def test_write(self): + rec = self.api.write( + level1_id=2, + spectra_id = "222", + region = "[12,13,24,24]", + prc_status = 3, + prc_time = '2021-06-05 11:12:13', + filename = "dddasd223234.fits", + file_path = "/opt/dddasd23.fits", + pipeline_id = "P2") + print('write:', rec) \ No newline at end of file