Skip to content
GitLab
Commit 434f1e57 authored by Wei Shoulin's avatar Wei Shoulin
Browse files

add level0_id

parent 4a427749
Hide whitespace changes
Inline Side-by-side
csst_dfs_api_local/common/db.sql
View file @ 434f1e57
......@@ -306,10 +306,12 @@ create table msc_level1_prc
create table msc_level2_data (
id integer PRIMARY KEY autoincrement,
level0_id VARCHAR(20) null,
level1_id int(20) not null,
data_type VARCHAR(64) not null,
filename VARCHAR(128) null,
file_path VARCHAR(256) null,
obs_time datetime null,
prc_status tinyint(1),
prc_time datetime null,
qc2_status tinyint(1),
......@@ -701,6 +703,7 @@ create table sls_level2_spectra
(
id integer PRIMARY KEY autoincrement,
spectra_id varchar(40),
level0_id VARCHAR(20) null,
level1_id int(20) not null,
region varchar(128),
filename varchar(128),
......
csst_dfs_api_local/msc/level2.py
View file @ 434f1e57
import os
import logging
import time, datetime
from astropy.io import fits
import shutil
from traceback import print_stack
......@@ -9,6 +10,7 @@ from ..common.utils import *
from csst_dfs_commons.models import Result
from csst_dfs_commons.models.common import from_dict_list
from csst_dfs_commons.models.msc import Level2Record, Level2CatalogRecord
from csst_dfs_commons.utils.fits import get_header_value
log = logging.getLogger('csst')
......@@ -25,6 +27,7 @@ class Level2DataApi(object):
''' retrieve level2 records from database
:param kwargs: Parameter dictionary, key items support:
level0_id: [str]
level1_id: [int]
data_type: [str]
create_time : (start, end),
......@@ -36,6 +39,7 @@ class Level2DataApi(object):
:returns: csst_dfs_common.models.Result
'''
try:
level0_id = get_parameter(kwargs, "level0_id")
level1_id = get_parameter(kwargs, "level1_id")
data_type = get_parameter(kwargs, "data_type")
create_time_start = get_parameter(kwargs, "create_time", [None, None])[0]
......@@ -49,8 +53,10 @@ class Level2DataApi(object):
sql_data = f"select * from msc_level2_data where 1=1"
sql_condition = ""
if level0_id:
sql_condition = f"{sql_condition} and level0_id='{level0_id}'"
if level1_id:
sql_condition = f"{sql_condition} and level1_id='{level1_id}'"
sql_condition = f"{sql_condition} and level1_id={level1_id}"
if data_type:
sql_condition = f"{sql_condition} and data_type='{data_type}'"
if create_time_start:
......@@ -157,13 +163,16 @@ class Level2DataApi(object):
''' insert a level2 record into database
parameter kwargs:
level0_id: [str]
level1_id: [int]
data_type : [str]
filename : [str]
file_path : [str]
file_path : [str]
obs_time : [str]
prc_status : [int]
prc_time : [str]
return csst_dfs_common.models.Result
'''
try:
......@@ -185,10 +194,24 @@ class Level2DataApi(object):
log.error(f'{rec.filename} has already been existed')
return Result.error(message=f'{rec.filename} has already been existed')
hdul = fits.open(rec.file_path)
header = hdul[0].header
obs_id = get_header_value("OBSID",header, "")
obs_id_str = "%09d" % (obs_id) if isinstance(obs_id, int) else obs_id
detector = get_header_value("DETECTOR", header, "")
if len(detector) > 2:
detector = detector[-2:]
obs_time = f"{get_header_value('DATE-OBS', header, '')} {get_header_value('TIME-OBS', header, '')}"
level0_id = f"{obs_id_str}{detector}"
if not rec.filename:
rec.filename = os.path.basename(rec.file_path)
self.db.execute(
'INSERT INTO msc_level2_data (level1_id,data_type,filename,file_path,qc2_status,prc_status,prc_time,create_time,pipeline_id) \
VALUES(?,?,?,?,?,?,?,?,?)',
(rec.level1_id, rec.data_type, rec.filename, rec.file_path, -1, rec.prc_status, rec.prc_time, format_time_ms(time.time()),rec.pipeline_id)
'INSERT INTO msc_level2_data (level0_id, level1_id,data_type,filename,file_path,obs_time,qc2_status,prc_status,prc_time,create_time,pipeline_id) \
VALUES(?,?,?,?,?,?,?,?,?,?,?)',
(level0_id, rec.level1_id, rec.data_type, rec.filename, rec.file_path, obs_time, -1, rec.prc_status, rec.prc_time, format_time_ms(time.time()),rec.pipeline_id)
)
self.db.end()
rec.id = self.db.last_row_id()
......
csst_dfs_api_local/sls/level2spectra.py
View file @ 434f1e57
......@@ -21,6 +21,7 @@ class Level2SpectraApi(object):
''' retrieve records from database
parameter kwargs:
level0_id: [str]
level1_id: [int]
spectra_id: [str]
create_time : (start, end),
......@@ -32,6 +33,7 @@ class Level2SpectraApi(object):
return: csst_dfs_common.models.Result
'''
try:
level0_id = get_parameter(kwargs, "level0_id", "")
level1_id = get_parameter(kwargs, "level1_id", 0)
spectra_id = get_parameter(kwargs, "spectra_id")
create_time_start = get_parameter(kwargs, "create_time", [None, None])[0]
......@@ -45,6 +47,8 @@ class Level2SpectraApi(object):
sql_data = f"select * from sls_level2_spectra where 1=1"
sql_condition = ""
if level0_id:
sql_condition = f"{sql_condition} and level0_id='{level0_id}'"
if level1_id > 0:
sql_condition = f"{sql_condition} and level1_id={level1_id}"
if spectra_id:
......@@ -153,6 +157,7 @@ class Level2SpectraApi(object):
''' insert a level1 record into database
parameter kwargs:
level0_id: [str]
level1_id: [int]
spectra_id : [str]
region : [str]
......@@ -167,6 +172,7 @@ class Level2SpectraApi(object):
try:
rec = Level2Spectra(
id = 0,
level0_id = get_parameter(kwargs, "level0_id"),
level1_id = get_parameter(kwargs, "level1_id"),
spectra_id = get_parameter(kwargs, "spectra_id"),
region = get_parameter(kwargs, "region"),
......@@ -185,9 +191,9 @@ class Level2SpectraApi(object):
return Result.error(message=f'{rec.filename} has already been existed')
self.db.execute(
'INSERT INTO sls_level2_spectra (level1_id,spectra_id,region,filename,file_path,qc1_status,prc_status,prc_time,create_time,pipeline_id) \
VALUES(?,?,?,?,?,?,?,?,?,?)',
(rec.level1_id, rec.spectra_id, rec.region, rec.filename, rec.file_path, -1, rec.prc_status, rec.prc_time, format_time_ms(time.time()),rec.pipeline_id,)
'INSERT INTO sls_level2_spectra (level0_id, level1_id,spectra_id,region,filename,file_path,qc1_status,prc_status,prc_time,create_time,pipeline_id) \
VALUES(?,?,?,?,?,?,?,?,?,?,?)',
(rec.level0_id, rec.level1_id, rec.spectra_id, rec.region, rec.filename, rec.file_path, -1, rec.prc_status, rec.prc_time, format_time_ms(time.time()),rec.pipeline_id,)
)
self.db.end()
rec.id = self.db.last_row_id()
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment