Skip to content
GitLab
Commit ea741fe5 authored by Wei Shoulin's avatar Wei Shoulin
Browse files

c3

parent 66ee8623
Hide whitespace changes
Inline Side-by-side
csst_dfs_api_local/__init__.py
View file @ ea741fe5
from csst_dfs_commons.logging import setup_logging
setup_logging()
\ No newline at end of file
csst_dfs_api_local/common/db.py
View file @ ea741fe5
......@@ -42,6 +42,9 @@ class DBClient(object):
for s in statements:
self.execute(s)
def last_row_id(self):
return self._cursor.lastrowid
def select_one(self, sql, param=()):
"""查询单个结果"""
_ = self.__execute(sql, param)
......
csst_dfs_api_local/common/db.sql
View file @ ea741fe5
drop table if exists ifs_level1_data;
drop table if exists ifs_level1_header;
drop table if exists msc_level1_data;
drop table if exists msc_level1_header;
drop table if exists t_cal2level0;
drop table if exists t_cal_header;
drop table if exists t_cal_merge;
drop table if exists t_detector;
drop table if exists t_detector_status;
drop table if exists t_facility_status;
drop table if exists t_guiding;
drop table if exists t_level0_data;
drop table if exists t_level0_header;
drop table if exists t_level0_prc;
drop table if exists t_module_status;
drop table if exists t_observation;
/*==============================================================*/
/* Table: ifs_level1_data */
/*==============================================================*/
create table ifs_level1_data
(
id integer PRIMARY KEY autoincrement,
raw_id int(20) not null,
data_type varchar(64) not null,
cor_sci_id int(20),
prc_params varchar(1024),
flat_id int(20),
dark_id int(20),
bias_id int(20),
lamp_id int(20),
arc_id int(20),
sky_id int(20),
filename varchar(128),
file_path varchar(256),
prc_status tinyint(1),
prc_time datetime,
qc1_status tinyint(1),
qc1_time datetime,
create_time datetime,
pipeline_id varchar(60)
);
/*==============================================================*/
/* Table: ifs_level1_header */
/*==============================================================*/
create table ifs_level1_header
(
id int(20) not null,
obs_time datetime,
exp_time float,
ra float,
"dec" float,
create_time datetime,
primary key (id)
);
/*==============================================================*/
/* Table: ifs_result_0_1 */
/* Table: msc_level1_data */
/*==============================================================*/
create table ifs_result_0_1 (
result0_id INT not null,
result1_id INT not null,
create_time DATETIME null,
primary key (result0_id, result1_id)
create table msc_level1_data
(
id integer PRIMARY KEY autoincrement,
raw_id int(20) not null,
data_type varchar(64) not null,
cor_sci_id int(20),
prc_params varchar(1024),
flat_id int(20),
dark_id int(20),
bias_id int(20),
filename varchar(128),
file_path varchar(256),
prc_status tinyint(1),
prc_time datetime,
qc1_status tinyint(1),
qc1_time datetime,
create_time datetime,
pipeline_id varchar(60)
);
/*==============================================================*/
/* Table: ifs_raw_ref */
/* Table: msc_level1_header */
/*==============================================================*/
create table ifs_raw_ref (
fit_id INT not null,
ref_id INT not null,
create_time DATETIME null,
primary key (fit_id, ref_id)
create table msc_level1_header
(
id int(20) not null,
obs_time datetime,
exp_time float,
ra float,
"dec" float,
create_time datetime,
primary key (id)
);
/*==============================================================*/
/* Table: ifs_rawfits */
/* Table: t_cal2level0 */
/*==============================================================*/
create table ifs_rawfits (
id integer PRIMARY KEY autoincrement,
filename VARCHAR(100) null,
obs_time INT null,
ccd_num INT null,
exp_time DATETIME null,
file_path VARCHAR(128) null,
qc0_status tinyint(1) null,
qc0_time DATETIME null,
prc_status tinyint(1) null,
prc_time DATETIME null,
create_time DATETIME null
create table t_cal2level0
(
merge_id int(20) not null,
level0_id int(20) not null,
primary key (merge_id, level0_id)
);
/*==============================================================*/
/* Table: ifs_ref_fits */
/* Table: t_cal_header */
/*==============================================================*/
create table ifs_ref_fits (
id integer PRIMARY KEY autoincrement,
filename VARCHAR(128) null,
obs_time INT null,
exp_time DATETIME null,
ccd_num INT null,
file_path VARCHAR(256) null,
ref_type VARCHAR(32) null,
create_time DATETIME null,
status tinyint(1) null
create table t_cal_header
(
id int(20) not null,
obs_time datetime,
exp_time float,
ra float,
"dec" float,
create_time datetime,
primary key (id)
);
/*==============================================================*/
/* Table: ifs_result_0 */
/* Table: t_cal_merge */
/*==============================================================*/
create table ifs_result_0 (
id integer PRIMARY KEY autoincrement,
filename VARCHAR(100) null,
raw_id INT null,
file_path VARCHAR(128) null,
create_time DATETIME null,
proc_type VARCHAR(32) null
create table t_cal_merge
(
id integer PRIMARY KEY autoincrement,
detector_no varchar(64) not null,
ref_type varchar(16),
obs_time datetime,
exp_time float,
filename varchar(128),
file_path varchar(256),
qc1_status tinyint(1),
qc1_time datetime,
prc_status tinyint(1),
prc_time datetime,
create_time datetime
);
/*==============================================================*/
/* Table: ifs_result_1 */
/* Table: t_detector */
/*==============================================================*/
create table ifs_result_1 (
id integer PRIMARY KEY autoincrement,
filename VARCHAR(100) null,
file_path VARCHAR(128) null,
create_time DATETIME null,
proc_type VARCHAR(32) null
create table t_detector
(
no varchar(64) not null,
detector_name varchar(256) not null,
module_id varchar(20),
filter_id varchar(20),
create_time datetime,
update_time datetime,
primary key (no)
);
/*==============================================================*/
/* Table: t_detector_status */
/*==============================================================*/
create table t_detector_status
(
id integer PRIMARY KEY autoincrement,
detector_no varchar(64) not null,
status varchar(256) not null,
status_time datetime,
create_time datetime
);
/*==============================================================*/
/* Table: t_facility_status */
/*==============================================================*/
create table t_facility_status
(
id integer PRIMARY KEY autoincrement,
status varchar(256) not null,
status_time datetime,
create_time datetime
);
/*==============================================================*/
/* Table: t_guiding */
/*==============================================================*/
create table t_guiding
(
id integer PRIMARY KEY autoincrement,
filename varbinary(128),
guiding_file_path varchar(256) not null,
guiding_no varchar(256),
create_time datetime
);
/*==============================================================*/
/* Table: t_level0_data */
/*==============================================================*/
create table t_level0_data
(
id integer PRIMARY KEY autoincrement,
obs_id int(20) not null,
detector_no varchar(64) not null,
obs_type varchar(16),
obs_time datetime,
exp_time float,
detector_status_id int(20),
filename varchar(128),
file_path varchar(256),
qc0_status tinyint(1),
qc0_time datetime,
prc_status tinyint(1),
prc_time datetime,
create_time datetime
);
/*==============================================================*/
/* Table: t_level0_header */
/*==============================================================*/
create table t_level0_header
(
id int(20) not null,
obs_time datetime,
exp_time float,
ra float,
"dec" float,
create_time datetime,
primary key (id)
);
/*==============================================================*/
/* Table: t_level0_prc */
/*==============================================================*/
create table t_level0_prc
(
id integer PRIMARY KEY autoincrement,
level0_id int(20) not null,
pipeline_id varchar(64) not null,
prc_module varchar(32) not null,
params_id varchar(256),
prc_status int(2),
prc_time datetime,
file_path varchar(256)
);
/*==============================================================*/
/* Table: t_module_status */
/*==============================================================*/
create table t_module_status
(
id integer PRIMARY KEY autoincrement,
module_id varbinary(20),
status varchar(256) not null,
status_time datetime,
create_time datetime
);
/*==============================================================*/
/* Table: t_observation */
/*==============================================================*/
create table t_observation
(
id integer PRIMARY KEY autoincrement,
obs_time datetime,
exp_time float,
module_id varchar(20),
obs_type varchar(16),
facility_status_id int(20),
module_status_id int(20),
qc0_status tinyint(1),
qc0_time datetime,
prc_status tinyint(1),
prc_time datetime,
create_time datetime,
import_status tinyint(1)
);
csst_dfs_api_local/common/ingest.py 0 → 100644
View file @ ea741fe5
import os, sys
import argparse
import logging
from astropy.io import fits
import datetime
import shutil
from csst_dfs_api_local.common.db import DBClient
log = logging.getLogger('csst-dfs-api-local')
def ingest():
db = DBClient()
parser = argparse.ArgumentParser(prog=f"{sys.argv[0]}", description="ingest the local files")
parser.add_argument('-i','--infile', dest="infile", help="a file or a directory")
parser.add_argument('-m', '--copyfiles', dest="copyfiles", action='store_true', default=False, help="move files after import")
args = parser.parse_args(sys.argv[1:])
import_root_dir = args.infile
if os.path.isfile(import_root_dir):
log.info(f"prepare import {import_root_dir}")
ingesst_one(import_root_dir, db, args.copyfiles)
if os.path.isdir(import_root_dir):
for (path, _, file_names) in os.walk(import_root_dir):
for filename in file_names:
if filename.find(".fits") > 0:
file_full_path = os.path.join(path, filename)
log.info(f"prepare import {file_full_path}")
ingesst_one(file_full_path, db, args.copyfiles)
db.close()
def ingesst_one(file_path, db, copyfiles):
dest_root_dir = os.getenv("CSST_LOCAL_FILE_ROOT", "/opt/temp/csst")
hdul = fits.open(file_path)
header = hdul[0].header
obs_id = header["OBSID"]
exp_start_time = f"{header['DATE-OBS']} {header['TIME-OBS']}"
exp_time = header['EXPTIME']
module_id = header["INSTRUME"]
obs_type = header["FILETYPE"]
qc0_status = 0
prc_status = 0
time_now = datetime.datetime.now()
create_time = time_now.strftime('%Y-%m-%d %H:%M:%S')
facility_status_id = 0
module_status_id = 0
existed = db.exists("select * from t_observation where id=?", (obs_id,))
if not existed:
db.execute("insert into t_observation \
(id,obs_time,exp_time,module_id,obs_type,facility_status_id, module_status_id, qc0_status, prc_status,create_time) \
values (?,?,?,?,?,?,?,?,?,?)",
(obs_id,exp_start_time,exp_time,module_id,obs_type,facility_status_id,module_status_id,qc0_status, prc_status,create_time))
#level0
detector = header["DETECTOR"]
filename = header["FILENAME"]
existed = db.exists(
"select * from t_level0_data where filename=?",
(filename,)
)
if existed:
log.warning('%s has already been imported' %(file_path, ))
db.end()
return
detector_status_id = 0
file_full_path = file_path
if copyfiles:
obs_id_str = "%07d" % (obs_id)
file_dir = f"{dest_root_dir}/{module_id}/{obs_type.upper()}/{header['EXPSTART']}/{obs_id_str}/MS"
if not os.path.exists(file_dir):
os.makedirs(file_dir)
file_full_path = f"{file_dir}/{filename}.fits"
c = db.execute("insert into t_level0_data \
(obs_id, detector_no, obs_type, obs_time, exp_time,detector_status_id, filename, file_path,qc0_status, prc_status,create_time) \
values (?,?,?,?,?,?,?,?,?,?,?)",
(obs_id, detector, obs_type, exp_start_time, exp_time, detector_status_id, filename, file_full_path, qc0_status, prc_status,create_time))
db.end()
level0_id = db.last_row_id()
#level0-header
ra_obj = header["RA_OBJ"]
dec_obj = header["DEC_OBJ"]
c = db.execute("insert into t_level0_header \
(id, obs_time, exp_time, ra, `dec`, create_time) \
values (?,?,?,?,?,?)",
(level0_id, exp_start_time, exp_time, ra_obj, dec_obj, create_time))
if copyfiles:
#copy files
shutil.copyfile(file_path, file_full_path)
db.end()
print(f"{file_path} imported")
if __name__ == "__main__":
ingest()
\ No newline at end of file
csst_dfs_api_local/facility/__init__.py 0 → 100644
View file @ ea741fe5
from .calmerge import CalMergeApi
from .level0 import Level0DataApi
from .detector import DetectorApi
from .level0prc import Level0PrcApi
from .observation import ObservationApi
\ No newline at end of file
csst_dfs_api_local/facility/calmerge.py 0 → 100644
View file @ ea741fe5
import os
import logging
import time, datetime
import shutil
from ..common.db import DBClient
from ..common.utils import *
from csst_dfs_commons.models import Result
from csst_dfs_commons.models.facility import CalMergeRecord
from csst_dfs_commons.models.common import from_dict_list
log = logging.getLogger('csst')
class CalMergeApi(object):
def __init__(self):
self.root_dir = os.getenv("CSST_LOCAL_FILE_ROOT", "/opt/temp/csst")
self.db = DBClient()
def find(self, **kwargs):
''' retrieve calibration merge records from database
parameter kwargs:
detector_no: [str]
ref_type: [str]
obs_time: (start,end)
qc1_status : [int]
prc_status : [int]
file_name: [str]
limit: limits returns the number of records,default 0:no-limit
return: csst_dfs_common.models.Result
'''
try:
detector_no = get_parameter(kwargs, "detector_no")
ref_type = get_parameter(kwargs, "ref_type")
exp_time_start = get_parameter(kwargs, "obs_time", [None, None])[0]
exp_time_end = get_parameter(kwargs, "obs_time", [None, None])[1]
qc1_status = get_parameter(kwargs, "qc1_status")
prc_status = get_parameter(kwargs, "prc_status")
file_name = get_parameter(kwargs, "file_name")
limit = get_parameter(kwargs, "limit", 0)
sql_count = "select count(*) as c from t_cal_merge where 1=1"
sql_data = f"select * from t_cal_merge where 1=1"
sql_condition = ""
if detector_no:
sql_condition = f"{sql_condition} and detector_no='{detector_no}'"
if ref_type:
sql_condition = f"{sql_condition} and ref_type='{ref_type}'"
if exp_time_start:
sql_condition = f"{sql_condition} and obs_time >='{exp_time_start}'"
if exp_time_end:
sql_condition = f"{sql_condition} and obs_time <='{exp_time_end}'"
if qc1_status:
sql_condition = f"{sql_condition} and qc1_status={qc1_status}"
if prc_status:
sql_condition = f"{sql_condition} and prc_status={prc_status}"
if file_name:
sql_condition = f" and filename={file_name}"
sql_count = f"{sql_count} {sql_condition}"
sql_data = f"{sql_data} {sql_condition}"
log.info(sql_count)
log.info(sql_data)
if limit > 0:
sql_data = f"{sql_data} limit {limit}"
totalCount = self.db.select_one(sql_count)
_, records = self.db.select_many(sql_data)
return Result.ok_data(data=from_dict_list(CalMergeRecord, records)).append("totalCount", totalCount['c'])
except Exception as e:
return Result.error(message=str(e))
def get(self, **kwargs):
''' fetch a record from database
parameter kwargs:
id : [int]
return csst_dfs_common.models.Result
'''
try:
id = get_parameter(kwargs, "id")
r = self.db.select_one(
"select * from t_cal_merge where id=?", (id,))
if r:
sql_get_level0_id = f"select level0_id from t_cal2level0 where merge_id={r['id']}"
_, records = self.db.select_many(sql_get_level0_id)
level0_ids = [r["level0_id"] for r in records]
rec = CalMergeRecord().from_dict(r)
rec.level0_ids = level0_ids
return Result.ok_data(data=rec)
else:
return Result.error(message=f"id:{id} not found")
except Exception as e:
log.error(e)
return Result.error(message=str(e))
def update_qc1_status(self, **kwargs):
''' update the status of reduction
parameter kwargs:
id : [int],
status : [int]
return csst_dfs_common.models.Result
'''
id = get_parameter(kwargs, "id")
status = get_parameter(kwargs, "status")
try:
existed = self.db.exists(
"select * from t_cal_merge where id=?",
(id,)
)
if not existed:
log.warning('%s not found' %(id, ))
return Result.error(message ='%s not found' %(id, ))
self.db.execute(
'update t_cal_merge set qc1_status=?, qc1_time=? where id=?',
(status, format_time_ms(time.time()), id)
)
self.db.end()
return Result.ok_data()
except Exception as e:
log.error(e)
return Result.error(message=e.message)
def update_proc_status(self, **kwargs):
''' update the status of reduction
parameter kwargs:
id : [int],
status : [int]
return csst_dfs_common.models.Result
'''
id = get_parameter(kwargs, "id")
status = get_parameter(kwargs, "status")
try:
existed = self.db.exists(
"select * from t_cal_merge where id=?",
(id,)
)
if not existed:
log.warning('%s not found' %(id, ))
return Result.error(message ='%s not found' %(id, ))
self.db.execute(
'update t_cal_merge set prc_status=?, prc_time=? where id=?',
(status, format_time_ms(time.time()), id)
)
self.db.end()
return Result.ok_data()
except Exception as e:
log.error(e)
return Result.error(message=e.message)
def write(self, **kwargs):
''' insert a calibration merge record into database
parameter kwargs:
id : [int]
detector_no : [str]
ref_type : [str]
obs_time : [str]
exp_time : [float]
prc_status : [int]
prc_time : [str]
filename : [str]
file_path : [str]
level0_ids : [list]
return csst_dfs_common.models.Result
'''
rec = CalMergeRecord(
id = 0,
detector_no = get_parameter(kwargs, "detector_no"),
ref_type = get_parameter(kwargs, "ref_type"),
obs_time = get_parameter(kwargs, "obs_time"),
exp_time = get_parameter(kwargs, "exp_time"),
filename = get_parameter(kwargs, "filename"),
file_path = get_parameter(kwargs, "file_path"),
prc_status = get_parameter(kwargs, "prc_status"),
prc_time = get_parameter(kwargs, "prc_time"),
level0_ids = get_parameter(kwargs, "level0_ids", [])
)
try:
self.db.execute(
'INSERT INTO t_cal_merge (detector_no,ref_type,obs_time,exp_time,filename,file_path,prc_status,prc_time,create_time) \
VALUES(?,?,?,?,?,?,?,?,?)',
(rec.detector_no, rec.ref_type, rec.obs_time, rec.exp_time, rec.filename, rec.file_path,rec.prc_status,rec.prc_time,format_time_ms(time.time()))
)
self.db.end()
rec.id = self.db.last_row_id()
sql_level0_ids = "insert into t_cal2level0 (merge_id,level0_id) values "
values = ["(%s,%s)"%(rec.id,i) for i in rec.level0_ids]
_ = self.db.execute(sql_level0_ids + ",".join(values))
self.db.end()
return Result.ok_data(data=rec)
except Exception as e:
log.error(e)
return Result.error(message=str(e))
\ No newline at end of file
csst_dfs_api_local/facility/detector.py 0 → 100644
View file @ ea741fe5
import os
import logging
import time, datetime
import shutil
from csst_dfs_commons.models import Result
from csst_dfs_commons.models.facility import Detector, DetectorStatus
from csst_dfs_commons.models.common import from_dict_list
from ..common.db import DBClient
from ..common.utils import *
log = logging.getLogger('csst')
class DetectorApi(object):
def __init__(self):
self.root_dir = os.getenv("CSST_LOCAL_FILE_ROOT", "/opt/temp/csst")
self.db = DBClient()
def find(self, **kwargs):
''' retrieve detector records from database
parameter kwargs:
module_id: [str]
key: [str]
return: csst_dfs_common.models.Result
'''
try:
module_id = get_parameter(kwargs, "module_id")
key = get_parameter(kwargs, "key","")
sql_data = f"select * from t_detector where 1=1"
sql_condition = ""
if module_id:
sql_condition = f"{sql_condition} and module_id='{module_id}'"
sql_condition = f"{sql_condition} and (`no` like ? or detector_name like ?)"
sql_data = f"{sql_data} {sql_condition}"
_, records = self.db.select_many(sql_data,(f'%{key}%',f'%{key}%'))
return Result.ok_data(data=from_dict_list(Detector, records)).append("totalCount", len(records))
except Exception as e:
return Result.error(message=str(e))
def get(self, **kwargs):
''' fetch a record from database
parameter kwargs:
no : [str]
return csst_dfs_common.models.Result
'''
try:
no = get_parameter(kwargs, "no")
r = self.db.select_one(
"select * from t_detector where no=?", (no,))
if r:
return Result.ok_data(data=Detector().from_dict(r))
else:
return Result.error(message=f"detector no:{no} not found")
except Exception as e:
log.error(e)
return Result.error(message=str(e))
def update(self, **kwargs):
''' update a detector by no
parameter kwargs:
no : [str],
detector_name : [str],
module_id : [str],
filter_id : [str]
return csst_dfs_common.models.Result
'''
try:
no = get_parameter(kwargs, "no")
result_get = self.get(no=no)
if not result_get.success:
return result_get
detector_name = get_parameter(kwargs, "detector_name", result_get.data.detector_name),
module_id = get_parameter(kwargs, "module_id", result_get.data.module_id),
filter_id = get_parameter(kwargs, "filter_id", result_get.data.filter_id)
sql_data = f"update t_detector set detector_name='{detector_name}',\
module_id='{module_id}',\
filter_id='{filter_id}',\
update_time=now() where `no`='{no}'"
_ = self.db.execute(sql_data)
self.db.end()
return Result.ok_data()
except Exception as e:
log.error(e)
return Result.error(message=str(e))
def delete(self, **kwargs):
''' delete a detector by no
parameter kwargs:
no : [str]
return csst_dfs_common.models.Result
'''
try:
no = get_parameter(kwargs, "no")
sql_data = f"delete from t_detector where `no`='{no}'"
_ = self.db.execute(sql_data)
self.db.end()
return Result.ok_data()
except Exception as e:
log.error(e)
return Result.error(message=str(e))
def write(self, **kwargs):
''' insert a detector record into database
parameter kwargs:
no : [str],
detector_name : [str],
module_id : [str],
filter_id : [str]
return csst_dfs_common.models.Result
'''
rec = Detector(
no = get_parameter(kwargs, "no"),
detector_name = get_parameter(kwargs, "detector_name"),
module_id = get_parameter(kwargs, "module_id"),
filter_id = get_parameter(kwargs, "filter_id"),
create_time = format_time_ms(time.time())
)
try:
existed = self.db.exists(
"select * from t_detector where no=?",
(rec.no,)
)
if existed:
log.warning('%s existed' %(rec.no, ))
return Result.error(message ='%s existed' %(rec.no, ))
self.db.execute(
'INSERT INTO t_detector (`no`,detector_name,module_id,filter_id,create_time) \
VALUES(?,?,?,?,?)',
(rec.no, rec.detector_name, rec.module_id, rec.filter_id,rec.create_time)
)
self.db.end()
return Result.ok_data(data=rec)
except Exception as e:
log.error(e)
return Result.error(message=str(e))
def find_status(self, **kwargs):
''' retrieve a detector status's from database
parameter kwargs:
detector_no: [str]
status_occur_time: (begin,end)
limit: limits returns the number of records,default 0:no-limit
return: csst_dfs_common.models.Result
'''
try:
detector_no = get_parameter(kwargs, "detector_no")
status_begin_time = get_parameter(kwargs, "status_occur_time", [None, None])[0]
status_end_time = get_parameter(kwargs, "status_occur_time", [None, None])[1]
limit = get_parameter(kwargs, "limit", 0)
sql_count = f"select count(*) as c from t_detector_status where detector_no='{detector_no}'"
sql_data = f"select * from t_detector_status where detector_no='{detector_no}'"
sql_condition = ""
if status_begin_time:
sql_condition = f"{sql_condition} and status_time >='{status_begin_time}'"
if status_end_time:
sql_condition = f"{sql_condition} and status_time <='{status_end_time}'"
sql_count = f"{sql_count} {sql_condition}"
sql_data = f"{sql_data} {sql_condition}"
if limit > 0:
sql_data = f"{sql_data} limit {limit}"
totalCount = self.mysql_client.select_one(sql_count)
_, records = self.mysql_client.select_many(sql_data)
return Result.ok_data(data=from_dict_list(DetectorStatus, records)).append("totalCount", totalCount['c'])
except Exception as e:
return Result.error(message=str(e))
def get_status(self, **kwargs):
''' fetch a record from database
parameter kwargs:
id : [int]
return csst_dfs_common.models.Result
'''
try:
id = get_parameter(kwargs, "id", -1)
r = self.db.select_one(
"select * from t_detector_status where id=?", (id,))
if r:
return Result.ok_data(data=DetectorStatus().from_dict(r))
else:
return Result.error(message=f"id:{id} not found")
except Exception as e:
log.error(e)
return Result.error(message=str(e))
def write_status(self, **kwargs):
''' insert a detector status into database
parameter kwargs:
detector_no : [str],
status : [str],
status_time : [str]
return csst_dfs_common.models.Result
'''
rec = DetectorStatus(
id = 0,
detector_no = get_parameter(kwargs, "detector_no"),
status = get_parameter(kwargs, "status"),
status_time = get_parameter(kwargs, "status_time")
)
try:
self.db.execute(
'INSERT INTO t_detector_status (detector_no,status,status_time,create_time) \
VALUES(?,?,?,?)',
(rec.detector_no, rec.status, rec.status_time,format_time_ms(time.time()))
)
self.db.end()
rec.id = self.db.last_row_id()
return Result.ok_data(data=rec)
except Exception as e:
log.error(e)
return Result.error(message=str(e))
\ No newline at end of file
csst_dfs_api_local/facility/level0.py 0 → 100644
View file @ ea741fe5
import os
import logging
import time, datetime
import shutil
from ..common.db import DBClient
from ..common.utils import *
from csst_dfs_commons.models import Result
from csst_dfs_commons.models.facility import Level0Record
from csst_dfs_commons.models.common import from_dict_list
log = logging.getLogger('csst')
class Level0DataApi(object):
def __init__(self):
self.root_dir = os.getenv("CSST_LOCAL_FILE_ROOT", "/opt/temp/csst")
self.db = DBClient()
def find(self, **kwargs):
''' retrieve level0 records from database
parameter kwargs:
obs_id: [int]
detector_no: [str]
obs_type: [str]
obs_time : (start, end),
qc0_status : [int],
prc_status : [int],
file_name: [str]
limit: limits returns the number of records,default 0:no-limit
return: csst_dfs_common.models.Result
'''
try:
obs_id = get_parameter(kwargs, "obs_id")
detector_no = get_parameter(kwargs, "detector_no")
obs_type = get_parameter(kwargs, "obs_type")
exp_time_start = get_parameter(kwargs, "obs_time", [None, None])[0]
exp_time_end = get_parameter(kwargs, "obs_time", [None, None])[1]
qc0_status = get_parameter(kwargs, "qc0_status")
prc_status = get_parameter(kwargs, "prc_status")
file_name = get_parameter(kwargs, "file_name")
limit = get_parameter(kwargs, "limit", 0)
sql_count = "select count(*) as c from t_level0_data where 1=1"
sql_data = f"select * from t_level0_data where 1=1"
sql_condition = ""
if obs_id > 0:
sql_condition = f"{sql_condition} and obs_id={obs_id}"
if detector_no:
sql_condition = f"{sql_condition} and detector_no='{detector_no}'"
if obs_type:
sql_condition = f"{sql_condition} and obs_type='{obs_type}'"
if exp_time_start:
sql_condition = f"{sql_condition} and obs_time >='{exp_time_start}'"
if exp_time_end:
sql_condition = f"{sql_condition} and obs_time <='{exp_time_end}'"
if qc0_status:
sql_condition = f"{sql_condition} and qc0_status={qc0_status}"
if prc_status:
sql_condition = f"{sql_condition} and prc_status={prc_status}"
if file_name:
sql_condition = f" and filename='{file_name}'"
sql_count = f"{sql_count} {sql_condition}"
sql_data = f"{sql_data} {sql_condition}"
if limit > 0:
sql_data = f"{sql_data} limit {limit}"
totalCount = self.db.select_one(sql_count)
_, records = self.db.select_many(sql_data)
return Result.ok_data(data=from_dict_list(Level0Record, records)).append("totalCount", totalCount['c'])
except Exception as e:
return Result.error(message=str(e))
def get(self, **kwargs):
''' fetch a record from database
parameter kwargs:
fits_id : [int]
return csst_dfs_common.models.Result
'''
try:
fits_id = get_parameter(kwargs, "fits_id", -1)
r = self.db.select_one(
"select * from t_level0_data where id=?", (fits_id,))
if r:
return Result.ok_data(data=Level0Record().from_dict(r))
else:
return Result.error(message=f"fits_id:{fits_id} not found")
except Exception as e:
log.error(e)
return Result.error(message=str(e))
def update_proc_status(self, **kwargs):
''' update the status of reduction
parameter kwargs:
fits_id : [int],
status : [int]
return csst_dfs_common.models.Result
'''
fits_id = get_parameter(kwargs, "fits_id")
status = get_parameter(kwargs, "status")
try:
existed = self.db.exists(
"select * from t_level0_data where id=?",
(fits_id,)
)
if not existed:
log.warning('%s not found' %(fits_id, ))
return Result.error(message ='%s not found' %(fits_id, ))
self.db.execute(
'update t_level0_data set prc_status=?, prc_time=? where id=?',
(status, format_time_ms(time.time()), fits_id)
)
self.db.end()
return Result.ok_data()
except Exception as e:
log.error(e)
return Result.error(message=str(e))
def update_qc0_status(self, **kwargs):
''' update the status of QC0
parameter kwargs:
fits_id : [int],
status : [int]
'''
fits_id = get_parameter(kwargs, "fits_id")
status = get_parameter(kwargs, "status")
try:
existed = self.db.exists(
"select * from t_level0_data where id=?",
(fits_id,)
)
if not existed:
log.warning('%s not found' %(fits_id, ))
return Result.error(message ='%s not found' %(fits_id, ))
self.db.execute(
'update t_level0_data set qc0_status=?, qc0_time=? where id=?',
(status, format_time_ms(time.time()), fits_id)
)
self.db.end()
return Result.ok_data()
except Exception as e:
log.error(e)
return Result.error(message=str(e))
def write(self, **kwargs):
''' insert a level0 data record into database
parameter kwargs:
obs_id = [int]
detector_no = [str]
obs_type = [str]
obs_time = [str]
exp_time = [int]
detector_status_id = [int]
filename = [str]
file_path = [str]
return: csst_dfs_common.models.Result
'''
rec = Level0Record(
obs_id = get_parameter(kwargs, "obs_id"),
detector_no = get_parameter(kwargs, "detector_no"),
obs_type = get_parameter(kwargs, "obs_type"),
obs_time = get_parameter(kwargs, "obs_time"),
exp_time = get_parameter(kwargs, "exp_time"),
detector_status_id = get_parameter(kwargs, "detector_status_id"),
filename = get_parameter(kwargs, "filename"),
file_path = get_parameter(kwargs, "file_path")
)
try:
existed = self.db.exists(
"select * from t_level0_data where filename=?",
(rec.filename,)
)
if existed:
log.warning('%s existed' %(rec.filename, ))
return Result.error(message ='%s existed' %(rec.filename, ))
self.db.execute(
'INSERT INTO t_level0_data (obs_id, detector_no, obs_type, obs_time, exp_time,detector_status_id, filename, file_path,qc0_status, prc_status,create_time) \
VALUES(?,?,?,?,?,?,?,?,?,?,?)',
(rec.obs_id, rec.detector_no, rec.obs_type, rec.obs_time, rec.exp_time, rec.detector_status_id, rec.filename, rec.file_path,0,0,format_time_ms(time.time()))
)
self.db.end()
rec.id = self.db.last_row_id()
return Result.ok_data(data=rec)
except Exception as e:
log.error(e)
return Result.error(message=str(e))
csst_dfs_api_local/facility/level0prc.py 0 → 100644
View file @ ea741fe5
import os
import logging
import time, datetime
import shutil
from ..common.db import DBClient
from ..common.utils import *
from csst_dfs_commons.models import Result
from csst_dfs_commons.models.facility import Level0PrcRecord
from csst_dfs_commons.models.common import from_dict_list
log = logging.getLogger('csst')
class Level0PrcApi(object):
def __init__(self):
self.root_dir = os.getenv("CSST_LOCAL_FILE_ROOT", "/opt/temp/csst")
self.db = DBClient()
def find(self, **kwargs):
''' retrieve level0 procedure records from database
parameter kwargs:
level0_id: [int]
pipeline_id: [str]
prc_module: [str]
prc_status : [int]
return: csst_dfs_common.models.Result
'''
try:
level0_id = get_parameter(kwargs, "level0_id")
pipeline_id = get_parameter(kwargs, "pipeline_id")
prc_module = get_parameter(kwargs, "prc_module")
prc_status = get_parameter(kwargs, "prc_status")
sql_data = f"select * from t_level0_prc"
sql_condition = f"where level0_id={level0_id}"
if pipeline_id:
sql_condition = sql_condition + " and pipeline_id='" + pipeline_id + "'"
if prc_module:
sql_condition = sql_condition + " and prc_module ='" + prc_module + "'"
if prc_status:
sql_condition = f"{sql_condition} and prc_status={prc_status}"
sql_data = f"{sql_data} {sql_condition}"
_, records = self.db.select_many(sql_data)
return Result.ok_data(data=from_dict_list(Level0PrcRecord, records)).append("totalCount", len(records))
except Exception as e:
return Result.error(message=str(e))
def update_proc_status(self, **kwargs):
''' update the status of reduction
parameter kwargs:
id : [int],
status : [int]
return csst_dfs_common.models.Result
'''
id = get_parameter(kwargs, "id")
status = get_parameter(kwargs, "status")
try:
existed = self.db.exists(
"select * from t_level0_prc where id=?",
(id,)
)
if not existed:
log.warning('%s not found' %(id, ))
return Result.error(message ='%s not found' %(id, ))
self.db.execute(
'update t_level0_prc set prc_status=?, prc_time=? where id=?',
(status, format_time_ms(time.time()), id)
)
self.db.end()
return Result.ok_data()
except Exception as e:
log.error(e)
return Result.error(message=str(e))
def write(self, **kwargs):
''' insert a level0 procedure record into database
parameter kwargs:
level0_id : [int]
pipeline_id : [str]
prc_module : [str]
params_id : [str]
prc_status : [int]
prc_time : [str]
file_path : [str]
return csst_dfs_common.models.Result
'''
rec = Level0PrcRecord(
id = 0,
level0_id = get_parameter(kwargs, "level0_id"),
pipeline_id = get_parameter(kwargs, "pipeline_id"),
prc_module = get_parameter(kwargs, "prc_module"),
params_id = get_parameter(kwargs, "params_id"),
prc_status = get_parameter(kwargs, "prc_status"),
prc_time = get_parameter(kwargs, "prc_time"),
file_path = get_parameter(kwargs, "file_path")
)
try:
self.db.execute(
'INSERT INTO t_level0_prc (level0_id,pipeline_id,prc_module, params_id,prc_status,prc_time,file_path) \
VALUES(?,?,?,?,?,?,?)',
(rec.level0_id, rec.pipeline_id, rec.prc_module, rec.params_id, rec.prc_status, rec.prc_time, rec.file_path)
)
self.db.end()
rec.id = self.db.last_row_id()
return Result.ok_data(data=rec)
except Exception as e:
log.error(e)
return Result.error(message=str(e))
csst_dfs_api_local/facility/observation.py 0 → 100644
View file @ ea741fe5
import os
import logging
import time, datetime
import shutil
from ..common.db import DBClient
from ..common.utils import *
from csst_dfs_commons.models import Result
from csst_dfs_commons.models.facility import Observation
from csst_dfs_commons.models.common import from_dict_list
log = logging.getLogger('csst')
class ObservationApi(object):
def __init__(self):
self.root_dir = os.getenv("CSST_LOCAL_FILE_ROOT", "/opt/temp/csst")
self.db = DBClient()
def find(self, **kwargs):
''' retrieve exposure records from database
parameter kwargs:
module_id: [str]
obs_type: [str]
obs_time : (start, end),
qc0_status : [int],
prc_status : [int],
limit: limits returns the number of records,default 0:no-limit
return: csst_dfs_common.models.Result
'''
try:
module_id = get_parameter(kwargs, "module_id")
obs_type = get_parameter(kwargs, "obs_type")
exp_time_start = get_parameter(kwargs, "obs_time", [None, None])[0]
exp_time_end = get_parameter(kwargs, "obs_time", [None, None])[1]
qc0_status = get_parameter(kwargs, "qc0_status")
prc_status = get_parameter(kwargs, "prc_status")
limit = get_parameter(kwargs, "limit", 0)
sql_count = "select count(*) as c from t_observation"
sql_data = f"select * from t_observation"
sql_condition = "where module_id='" + module_id + "'"
if obs_type:
sql_condition = sql_condition + " and obs_type='" + obs_type + "'"
if exp_time_start:
sql_condition = sql_condition + " and obs_time >='" + exp_time_start + "'"
if exp_time_end:
sql_condition = sql_condition + " and obs_time <='" + exp_time_end + "'"
if qc0_status:
sql_condition = f"{sql_condition} and qc0_status={qc0_status}"
if prc_status:
sql_condition = f"{sql_condition} and prc_status={prc_status}"
sql_count = f"{sql_count} {sql_condition}"
sql_data = f"{sql_data} {sql_condition}"
if limit > 0:
sql_data = f"{sql_data} limit {limit}"
totalCount = self.db.select_one(sql_count)
_, records = self.db.select_many(sql_data)
return Result.ok_data(data=from_dict_list(Observation, records)).append("totalCount", totalCount['c'])
except Exception as e:
return Result.error(message=str(e))
def get(self, **kwargs):
'''
parameter kwargs:
obs_id = [int]
return dict or None
'''
try:
obs_id = get_parameter(kwargs, "obs_id", -1)
r = self.db.select_one(
"select * from t_observation where id=?", (obs_id,))
if r:
return Result.ok_data(data=Observation().from_dict(r))
else:
return Result.error(message=f"obs_id:{obs_id} not found")
except Exception as e:
log.error(e)
return Result.error(message=str(e))
def update_proc_status(self, **kwargs):
''' update the status of reduction
parameter kwargs:
obs_id : [int],
status : [int]
return csst_dfs_common.models.Result
'''
obs_id = get_parameter(kwargs, "obs_id")
status = get_parameter(kwargs, "status")
try:
existed = self.db.exists(
"select * from t_observation where id=?",
(obs_id,)
)
if not existed:
log.warning('%s not found' %(obs_id, ))
return Result.error(message ='%s not found' %(obs_id, ))
self.db.execute(
'update t_observation set prc_status=?, prc_time=? where id=?',
(status, format_time_ms(time.time()), obs_id)
)
self.db.end()
return Result.ok_data()
except Exception as e:
log.error(e)
return Result.error(message=str(e))
def update_qc0_status(self, **kwargs):
''' update the status of QC0
parameter kwargs:
obs_id : [int],
status : [int]
'''
obs_id = get_parameter(kwargs, "obs_id")
status = get_parameter(kwargs, "status")
try:
existed = self.db.exists(
"select * from t_observation where id=?",
(obs_id,)
)
if not existed:
log.warning('%s not found' %(obs_id, ))
return Result.error(message ='%s not found' %(obs_id, ))
self.db.execute(
'update t_observation set qc0_status=?, qc0_time=? where id=?',
(status, format_time_ms(time.time()), obs_id)
)
self.db.end()
return Result.ok_data()
except Exception as e:
log.error(e)
return Result.error(message=str(e))
def write(self, **kwargs):
''' insert a observational record into database
parameter kwargs:
obs_id = [int]
obs_time = [str]
exp_time = [int]
module_id = [str]
obs_type = [str]
facility_status_id = [int]
module_status_id = [int]
return: csst_dfs_common.models.Result
'''
obs_id = get_parameter(kwargs, "obs_id", 0)
obs_time = get_parameter(kwargs, "obs_time")
exp_time = get_parameter(kwargs, "exp_time")
module_id = get_parameter(kwargs, "module_id")
obs_type = get_parameter(kwargs, "obs_type")
facility_status_id = get_parameter(kwargs, "facility_status_id")
module_status_id = get_parameter(kwargs, "module_status_id")
try:
if obs_id == 0:
r = self.db.select_one("select max(id) as max_id from t_observation")
obs_id = r["max_id"]+1
existed = self.db.exists(
"select * from t_observation where id=?",
(obs_id,)
)
if existed:
log.warning('%s existed' %(obs_id, ))
return Result.error(message ='%s existed' %(obs_id, ))
self.db.execute(
'INSERT INTO t_observation (id,obs_time,exp_time,module_id,obs_type,facility_status_id, module_status_id, qc0_status,create_time) \
VALUES(?,?,?,?,?,?,?,?,?)',
(obs_id, obs_time, exp_time, module_id, obs_type, facility_status_id, module_status_id,0,format_time_ms(time.time()))
)
self.db.end()
return self.get(obs_id = obs_id)
except Exception as e:
log.error(e)
return Result.error(message=str(e))
\ No newline at end of file
csst_dfs_api_local/ifs/__init__.py
View file @ ea741fe5
from .fits import FitsApi
from .reffits import RefFitsApi
from .result0 import Result0Api
from .result1 import Result1Api
def ingest():
import os
root_dir = os.getenv("CSST_LOCAL_FILE_ROOT", "/opt/temp/csst")
paths = {}
fitsApi = FitsApi()
refApi = RefFitsApi()
for (path, _, file_names) in os.walk(root_dir):
for filename in file_names:
if filename.find(".fits") > 0:
file_full_path = os.path.join(path, filename)
file_type = 'None'
if 'obs' in filename.lower():
file_type = 'obs'
elif 'flat' in filename.lower():
file_type = 'flat'
elif 'bias' in filename.lower():
file_type = 'bias'
elif 'hgar' in filename.lower():
file_type = 'arc'
elif 'sky' in filename.lower():
file_type = 'sky'
file_path = file_full_path.replace(root_dir, '')
if file_path.index("/") == 0:
file_path = file_path[1:]
if file_type in ['obs']:
fitsApi.import2db(file_path = file_path)
print("%s [type:%s] imported" %(file_full_path, file_type))
if file_type in ['flat', 'bias', 'arc','hgar', 'sky']:
refApi.import2db(file_path = file_path)
print("%s [type:%s] imported" %(file_full_path, file_type))
return paths
from .level1 import Level1DataApi
\ No newline at end of file
csst_dfs_api_local/ifs/level1.py 0 → 100644
View file @ ea741fe5
import os
import logging
import time, datetime
import shutil
from ..common.db import DBClient
from ..common.utils import *
from csst_dfs_commons.models import Result
from csst_dfs_commons.models.ifs import Level1Record
from csst_dfs_commons.models.common import from_dict_list
log = logging.getLogger('csst')
class Level1DataApi(object):
def __init__(self, sub_system = "ifs"):
self.sub_system = sub_system
self.root_dir = os.getenv("CSST_LOCAL_FILE_ROOT", "/opt/temp/csst")
self.db = DBClient()
def find(self, **kwargs):
''' retrieve level1 records from database
parameter kwargs:
raw_id: [int]
data_type: [str]
obs_type: [str]
create_time : (start, end),
qc1_status : [int],
prc_status : [int],
filename: [str]
limit: limits returns the number of records,default 0:no-limit
return: csst_dfs_common.models.Result
'''
try:
raw_id = get_parameter(kwargs, "raw_id")
data_type = get_parameter(kwargs, "data_type")
create_time_start = get_parameter(kwargs, "create_time", [None, None])[0]
create_time_end = get_parameter(kwargs, "create_time", [None, None])[1]
qc1_status = get_parameter(kwargs, "qc1_status")
prc_status = get_parameter(kwargs, "prc_status")
filename = get_parameter(kwargs, "filename")
limit = get_parameter(kwargs, "limit", 0)
sql_count = "select count(*) as c from ifs_level1_data where 1=1"
sql_data = f"select * from ifs_level1_data where 1=1"
sql_condition = ""
if raw_id:
sql_condition = f"{sql_condition} and raw_id={raw_id}"
if data_type:
sql_condition = f"{sql_condition} and data_type='{data_type}'"
if create_time_start:
sql_condition = f"{sql_condition} and create_time >='{create_time_start}'"
if create_time_end:
sql_condition = f"{sql_condition} and create_time <='{create_time_end}'"
if qc1_status:
sql_condition = f"{sql_condition} and qc1_status={qc1_status}"
if prc_status:
sql_condition = f"{sql_condition} and prc_status={prc_status}"
if filename:
sql_condition = f" and filename='{filename}'"
sql_count = f"{sql_count} {sql_condition}"
sql_data = f"{sql_data} {sql_condition}"
if limit > 0:
sql_data = f"{sql_data} limit {limit}"
totalCount = self.db.select_one(sql_count)
_, recs = self.db.select_many(sql_data)
return Result.ok_data(data=from_dict_list(Level1Record, recs)).append("totalCount", totalCount['c'])
except Exception as e:
return Result.error(message=e.message)
def get(self, **kwargs):
'''
parameter kwargs:
id = [int]
return dict or None
'''
try:
fits_id = get_parameter(kwargs, "id", -1)
r = self.db.select_one(
"select * from ifs_level1_data where id=?", (fits_id,))
if r:
return Result.ok_data(data=Level1Record().from_dict(r))
else:
return Result.error(message=f"id:{fits_id} not found")
except Exception as e:
log.error(e)
return Result.error(message=e.message)
def update_proc_status(self, **kwargs):
''' update the status of reduction
parameter kwargs:
id : [int],
status : [int]
return csst_dfs_common.models.Result
'''
fits_id = get_parameter(kwargs, "id")
status = get_parameter(kwargs, "status")
try:
existed = self.db.exists(
"select * from ifs_level1_data where id=?",
(fits_id,)
)
if not existed:
log.warning('%s not found' %(fits_id, ))
return Result.error(message ='%s not found' %(fits_id, ))
self.db.execute(
'update ifs_level1_data set prc_status=?, prc_time=? where id=?',
(status, format_time_ms(time.time()), fits_id)
)
self.db.end()
return Result.ok_data()
except Exception as e:
log.error(e)
return Result.error(message=e.message)
def update_qc1_status(self, **kwargs):
''' update the status of QC1
parameter kwargs:
id : [int],
status : [int]
'''
fits_id = get_parameter(kwargs, "id")
status = get_parameter(kwargs, "status")
try:
existed = self.db.exists(
"select * from ifs_level1_data where id=?",
(fits_id,)
)
if not existed:
log.warning('%s not found' %(fits_id, ))
return Result.error(message ='%s not found' %(fits_id, ))
self.db.execute(
'update ifs_level1_data set qc1_status=?, qc1_time=? where id=?',
(status, format_time_ms(time.time()), fits_id)
)
self.db.end()
return Result.ok_data()
except Exception as e:
log.error(e)
return Result.error(message=e.message)
def write(self, **kwargs):
''' insert a level1 record into database
parameter kwargs:
raw_id : [int]
data_type : [str]
cor_sci_id : [int]
prc_params : [str]
flat_id : [int]
dark_id : [int]
bias_id : [int]
lamp_id : [int]
arc_id : [int]
sky_id : [int]
filename : [str]
file_path : [str]
prc_status : [int]
prc_time : [str]
pipeline_id : [str]
return csst_dfs_common.models.Result
'''
try:
rec = Level1Record(
id = 0,
raw_id = get_parameter(kwargs, "raw_id"),
data_type = get_parameter(kwargs, "data_type"),
cor_sci_id = get_parameter(kwargs, "cor_sci_id"),
prc_params = get_parameter(kwargs, "prc_params"),
flat_id = get_parameter(kwargs, "flat_id"),
dark_id = get_parameter(kwargs, "dark_id"),
bias_id = get_parameter(kwargs, "bias_id"),
lamp_id = get_parameter(kwargs, "lamp_id"),
arc_id = get_parameter(kwargs, "arc_id"),
sky_id = get_parameter(kwargs, "sky_id"),
filename = get_parameter(kwargs, "filename"),
file_path = get_parameter(kwargs, "file_path"),
prc_status = get_parameter(kwargs, "prc_status"),
prc_time = get_parameter(kwargs, "prc_time"),
pipeline_id = get_parameter(kwargs, "pipeline_id")
)
existed = self.db.exists(
"select * from ifs_level1_data where filename=?",
(rec.filename,)
)
if existed:
log.error(f'{rec.filename} has already been existed')
return Result.error(message=f'{rec.filename} has already been existed')
self.db.execute(
'INSERT INTO ifs_level1_data (raw_id,data_type,cor_sci_id,prc_params,flat_id,dark_id,bias_id,lamp_id,arc_id,sky_id,filename,file_path,qc1_status,prc_status,create_time,pipeline_id) \
VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)',
(rec.raw_id, rec.data_type, rec.cor_sci_id, rec.prc_params, rec.flat_id, rec.dark_id, rec.bias_id, rec.lamp_id, rec.arc_id, rec.sky_id, rec.filename, rec.file_path, 0, 0, format_time_ms(time.time()),rec.pipeline_id,)
)
self.db.end()
rec.id = self.db.last_row_id()
return Result.ok_data(data=rec)
except Exception as e:
log.error(e)
return Result.error(message=e.message)
\ No newline at end of file
csst_dfs_api_local/ifs/result1.py deleted 100644 → 0
View file @ 66ee8623
import os
import logging
import time, datetime
import shutil
from ..common.db import DBClient
from ..common.utils import *
log = logging.getLogger('csst')
class Result1Api(object):
def __init__(self, sub_system = "ifs"):
self.sub_system = sub_system
self.root_dir = os.getenv("CSST_LOCAL_FILE_ROOT", "/opt/temp/csst")
self.check_dir()
self.db = DBClient()
def check_dir(self):
if not os.path.exists(self.root_dir):
os.mkdir(self.root_dir)
log.info("using [%s] as root directory", self.root_dir)
if not os.path.exists(os.path.join(self.root_dir, "results1")):
os.mkdir(os.path.join(self.root_dir, "results1"))
def find(self, **kwargs):
'''
parameter kwargs:
file_name = [str],
proc_type = [str]
return list of level 1 record
'''
paths = []
file_name = get_parameter(kwargs, "file_name")
proc_type = get_parameter(kwargs, "proc_type", "default")
sql = ["select * from ifs_result_1 where proc_type='" + proc_type + "'"]
if file_name:
sql = ["select * from ifs_result_1 where filename='" + file_name + "'"]
_, recs = self.db.select_many("".join(sql))
for r in recs:
r['file_path'] = os.path.join(self.root_dir, r['file_path'])
return recs
def get(self, **kwargs):
'''
parameter kwargs:
fits_id = [int]
return dict or None
'''
fits_id = get_parameter(kwargs, "fits_id", -1)
r = self.db.select_one(
"select * from ifs_result_1 where id=?", (fits_id,))
_, result0s = self.db.select_many(
"select result0_id, create_time from ifs_result_0_1 where result1_id=?", (fits_id,))
if r:
r['file_path'] = os.path.join(self.root_dir, r['file_path'])
return r, result0s
def read(self, **kwargs):
''' yield bytes of fits file
parameter kwargs:
fits_id = [int],
file_path = [str],
chunk_size = [int] default 20480
yield bytes of fits file
'''
fits_id = get_parameter(kwargs, "fits_id")
file_path = get_parameter(kwargs, "file_path")
if fits_id is None and file_path is None:
raise Exception("fits_id or file_path need to be defined")
if fits_id is not None:
r = self.db.select_one(
"select * from ifs_result_1 where id=?", (fits_id))
if r is not None:
file_path = r["file_path"]
if file_path is not None:
chunk_size = get_parameter(kwargs, "chunk_size", 20480)
return yield_file_bytes(os.path.join(self.root_dir, file_path), chunk_size)
def write(self, **kwargs):
''' copy a local level 1 file to file storage, and insert a record into database
parameter kwargs:
file_path = [str],
proc_type = [str],
result0_ids = [list]
insert into database
'''
file_path = get_parameter(kwargs, "file_path")
proc_type = get_parameter(kwargs, "proc_type", "default")
result0_ids = get_parameter(kwargs, "result0_ids", [])
if file_path is None:
raise Exception("file_path need to be defined")
new_file_dir = create_dir(os.path.join(self.root_dir, "results1"),
self.sub_system,
"/".join([str(datetime.now().year),"%02d"%(datetime.now().month),"%02d"%(datetime.now().day)]))
file_basename = os.path.basename(file_path)
new_file_path = os.path.join(new_file_dir, file_basename)
shutil.copyfile(file_path, new_file_path)
file_path = new_file_path.replace(self.root_dir, '')
if file_path.index("/") == 0:
file_path = file_path[1:]
self.db.execute(
'INSERT INTO ifs_result_1 (filename, file_path, proc_type, create_time) \
VALUES(?,?,?,?)',
(file_basename, file_path, proc_type, format_time_ms(time.time()),)
)
self.db.end()
result1_id = 1
for id0 in result0_ids:
self.db.execute(
'INSERT INTO ifs_result_0_1 (result0_id, result1_id, create_time) \
VALUES(?,?,?)',
(id0, result1_id, format_time_ms(time.time()))
)
self.db.end()
log.info("result1 fits %s imported.", file_path)
return new_file_path
\ No newline at end of file
csst_dfs_api_local/msc/__init__.py 0 → 100644
View file @ ea741fe5
from .level1 import Level1DataApi
\ No newline at end of file
csst_dfs_api_local/msc/level1.py 0 → 100644
View file @ ea741fe5
import os
import logging
import time, datetime
import shutil
from ..common.db import DBClient
from ..common.utils import *
from csst_dfs_commons.models import Result
from csst_dfs_commons.models.msc import Level1Record
from csst_dfs_commons.models.common import from_dict_list
log = logging.getLogger('csst')
class Level1DataApi(object):
def __init__(self, sub_system = "msc"):
self.sub_system = sub_system
self.root_dir = os.getenv("CSST_LOCAL_FILE_ROOT", "/opt/temp/csst")
self.db = DBClient()
def find(self, **kwargs):
''' retrieve level1 records from database
parameter kwargs:
raw_id: [int]
data_type: [str]
obs_type: [str]
create_time : (start, end),
qc1_status : [int],
prc_status : [int],
filename: [str]
limit: limits returns the number of records,default 0:no-limit
return: csst_dfs_common.models.Result
'''
try:
raw_id = get_parameter(kwargs, "raw_id")
data_type = get_parameter(kwargs, "data_type")
create_time_start = get_parameter(kwargs, "create_time", [None, None])[0]
create_time_end = get_parameter(kwargs, "create_time", [None, None])[1]
qc1_status = get_parameter(kwargs, "qc1_status")
prc_status = get_parameter(kwargs, "prc_status")
filename = get_parameter(kwargs, "filename")
limit = get_parameter(kwargs, "limit", 0)
sql_count = "select count(*) as c from msc_level1_data where 1=1"
sql_data = f"select * from msc_level1_data where 1=1"
sql_condition = ""
if raw_id:
sql_condition = f"{sql_condition} and raw_id={raw_id}"
if data_type:
sql_condition = f"{sql_condition} and data_type='{data_type}'"
if create_time_start:
sql_condition = f"{sql_condition} and create_time >='{create_time_start}'"
if create_time_end:
sql_condition = f"{sql_condition} and create_time <='{create_time_end}'"
if qc1_status:
sql_condition = f"{sql_condition} and qc1_status={qc1_status}"
if prc_status:
sql_condition = f"{sql_condition} and prc_status={prc_status}"
if filename:
sql_condition = f" and filename='{filename}'"
sql_count = f"{sql_count} {sql_condition}"
sql_data = f"{sql_data} {sql_condition}"
if limit > 0:
sql_data = f"{sql_data} limit {limit}"
totalCount = self.db.select_one(sql_count)
_, recs = self.db.select_many(sql_data)
return Result.ok_data(data=from_dict_list(Level1Record, recs)).append("totalCount", totalCount['c'])
except Exception as e:
return Result.error(message=str(e))
def get(self, **kwargs):
'''
parameter kwargs:
id = [int]
return dict or None
'''
try:
fits_id = get_parameter(kwargs, "id", -1)
r = self.db.select_one(
"select * from msc_level1_data where id=?", (fits_id,))
if r:
return Result.ok_data(data=Level1Record().from_dict(r))
else:
return Result.error(message=f"id:{fits_id} not found")
except Exception as e:
log.error(e)
return Result.error(message=str(e))
def update_proc_status(self, **kwargs):
''' update the status of reduction
parameter kwargs:
id : [int],
status : [int]
return csst_dfs_common.models.Result
'''
fits_id = get_parameter(kwargs, "id")
status = get_parameter(kwargs, "status")
try:
existed = self.db.exists(
"select * from msc_level1_data where id=?",
(fits_id,)
)
if not existed:
log.warning('%s not found' %(fits_id, ))
return Result.error(message ='%s not found' %(fits_id, ))
self.db.execute(
'update msc_level1_data set prc_status=?, prc_time=? where id=?',
(status, format_time_ms(time.time()), fits_id)
)
self.db.end()
return Result.ok_data()
except Exception as e:
log.error(e)
return Result.error(message=str(e))
def update_qc1_status(self, **kwargs):
''' update the status of QC1
parameter kwargs:
id : [int],
status : [int]
'''
fits_id = get_parameter(kwargs, "id")
status = get_parameter(kwargs, "status")
try:
existed = self.db.exists(
"select * from msc_level1_data where id=?",
(fits_id,)
)
if not existed:
log.warning('%s not found' %(fits_id, ))
return Result.error(message ='%s not found' %(fits_id, ))
self.db.execute(
'update msc_level1_data set qc1_status=?, qc1_time=? where id=?',
(status, format_time_ms(time.time()), fits_id)
)
self.db.end()
return Result.ok_data()
except Exception as e:
log.error(e)
return Result.error(message=str(e))
def write(self, **kwargs):
''' insert a level1 record into database
parameter kwargs:
raw_id : [int]
data_type : [str]
cor_sci_id : [int]
prc_params : [str]
flat_id : [int]
dark_id : [int]
bias_id : [int]
lamp_id : [int]
arc_id : [int]
sky_id : [int]
filename : [str]
file_path : [str]
prc_status : [int]
prc_time : [str]
pipeline_id : [str]
return csst_dfs_common.models.Result
'''
try:
rec = Level1Record(
id = 0,
raw_id = get_parameter(kwargs, "raw_id"),
data_type = get_parameter(kwargs, "data_type"),
cor_sci_id = get_parameter(kwargs, "cor_sci_id"),
prc_params = get_parameter(kwargs, "prc_params"),
flat_id = get_parameter(kwargs, "flat_id"),
dark_id = get_parameter(kwargs, "dark_id"),
bias_id = get_parameter(kwargs, "bias_id"),
filename = get_parameter(kwargs, "filename"),
file_path = get_parameter(kwargs, "file_path"),
prc_status = get_parameter(kwargs, "prc_status"),
prc_time = get_parameter(kwargs, "prc_time"),
pipeline_id = get_parameter(kwargs, "pipeline_id")
)
existed = self.db.exists(
"select * from msc_level1_data where filename=?",
(rec.filename,)
)
if existed:
log.error(f'{rec.filename} has already been existed')
return Result.error(message=f'{rec.filename} has already been existed')
self.db.execute(
'INSERT INTO msc_level1_data (raw_id,data_type,cor_sci_id,prc_params,flat_id,dark_id,bias_id,filename,file_path,qc1_status,prc_status,create_time,pipeline_id) \
VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?)',
(rec.raw_id, rec.data_type, rec.cor_sci_id, rec.prc_params, rec.flat_id, rec.dark_id, rec.bias_id, rec.filename, rec.file_path, 0, 0, format_time_ms(time.time()),rec.pipeline_id,)
)
self.db.end()
rec.id = self.db.last_row_id()
return Result.ok_data(data=rec)
except Exception as e:
log.error(e)
return Result.error(message=str(e))
\ No newline at end of file
setup.cfg
View file @ ea741fe5
......@@ -27,4 +27,4 @@ install_requires =
csst_dfs_api_local.common = *.sql
[options.entry_points]
console_scripts =
csst-dfs-ifs-local-ingest = csst_dfs_api_local.ifs:ingest
\ No newline at end of file
csst-dfs-ingest = csst_dfs_api_local.common.ingest:ingest
\ No newline at end of file
tests/test_common_catalog.py
View file @ ea741fe5
......@@ -10,5 +10,5 @@ class CommonEphemTestCase(unittest.TestCase):
self.api = CatalogApi()
def test_gaia3_query(self):
result = self.api.gaia3_query(ra=56.234, dec=14.4665, radius=1, min_mag=-1, max_mag=-1, obstime=-1, limit=2)
result = self.api.gaia3_query(ra=54.234, dec=13.4665, radius=1, min_mag=-1, max_mag=-1, obstime=-1, limit=2)
print('return:', result)
\ No newline at end of file
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment