Skip to content
GitLab
Commit 52824ecc authored by Wei Shoulin's avatar Wei Shoulin
Browse files

ifs ver

parent ed6639e4
Hide whitespace changes
Inline Side-by-side
csst_dfs_api_local/ifs/ingest.py
View file @ 52824ecc
...@@ -7,6 +7,7 @@ import shutil ...@@ -7,6 +7,7 @@ import shutil
from csst_dfs_api_local.common.db import DBClient from csst_dfs_api_local.common.db import DBClient
from csst_dfs_commons.utils.fits import get_header_value from csst_dfs_commons.utils.fits import get_header_value
from csst_dfs_commons.models.ifs import Level0Record
log = logging.getLogger('csst-dfs-api-local') log = logging.getLogger('csst-dfs-api-local')
...@@ -53,7 +54,8 @@ def ingest_one(file_path, db, copyfiles): ...@@ -53,7 +54,8 @@ def ingest_one(file_path, db, copyfiles):
module_id = header["INSTRUME"] module_id = header["INSTRUME"]
obs_type = header["OBSTYPE"] obs_type = header["OBSTYPE"]
object_name = header["OBJECT"] object_name = get_header_value("OBJECT", header, "-")
qc0_status = -1 qc0_status = -1
prc_status = -1 prc_status = -1
time_now = datetime.datetime.now() time_now = datetime.datetime.now()
...@@ -71,8 +73,8 @@ def ingest_one(file_path, db, copyfiles): ...@@ -71,8 +73,8 @@ def ingest_one(file_path, db, copyfiles):
db.end() db.end()
#level0 #level0
detector = get_header_value("DETNAM", header1, "-") detector = get_header_value("DETNAM", header1, "-")
filename = header["FILENAME"] filename = get_header_value("FILENAME", header, os.path.basename(file_path))
version = get_header_value("IMG_VER", header, "-") version = get_header_value("IMG_VER", header1, "-")
existed = db.exists( existed = db.exists(
"select * from ifs_level0_data where filename=?", "select * from ifs_level0_data where filename=?",
...@@ -116,7 +118,21 @@ def ingest_one(file_path, db, copyfiles): ...@@ -116,7 +118,21 @@ def ingest_one(file_path, db, copyfiles):
db.end() db.end()
rec = Level0Record(
id = level0_id_id,
level0_id = level0_id,
obs_id = obs_id,
detector_no = detector,
obs_type = obs_type,
obs_time = exp_start_time,
exp_time = exp_time,
detector_status_id = detector_status_id,
filename = filename,
file_path = file_full_path
)
print(f"{file_path} imported") print(f"{file_path} imported")
return rec
if __name__ == "__main__": if __name__ == "__main__":
ingest() ingest()
\ No newline at end of file
csst_dfs_api_local/ifs/level0.py
View file @ 52824ecc
...@@ -8,6 +8,7 @@ from ..common.utils import * ...@@ -8,6 +8,7 @@ from ..common.utils import *
from csst_dfs_commons.models import Result from csst_dfs_commons.models import Result
from csst_dfs_commons.models.ifs import Level0Record from csst_dfs_commons.models.ifs import Level0Record
from csst_dfs_commons.models.common import from_dict_list from csst_dfs_commons.models.common import from_dict_list
from .ingest import ingest_one
log = logging.getLogger('csst') log = logging.getLogger('csst')
...@@ -201,46 +202,16 @@ class Level0DataApi(object): ...@@ -201,46 +202,16 @@ class Level0DataApi(object):
''' insert a level0 data record into database ''' insert a level0 data record into database
parameter kwargs: parameter kwargs:
obs_id = [str] file_path = [str],
detector_no = [str] copyfiles = [boolean]
obs_type = [str]
obs_time = [str]
exp_time = [int]
detector_status_id = [int]
filename = [str]
file_path = [str]
return: csst_dfs_common.models.Result return: csst_dfs_common.models.Result
''' '''
rec = Level0Record(
obs_id = get_parameter(kwargs, "obs_id"),
detector_no = get_parameter(kwargs, "detector_no"),
obs_type = get_parameter(kwargs, "obs_type"),
obs_time = get_parameter(kwargs, "obs_time"),
exp_time = get_parameter(kwargs, "exp_time"),
detector_status_id = get_parameter(kwargs, "detector_status_id"),
filename = get_parameter(kwargs, "filename"),
file_path = get_parameter(kwargs, "file_path")
)
rec.level0_id = f"{rec.obs_id}{rec.detector_no}"
try:
existed = self.db.exists(
"select * from ifs_level0_data where filename=?",
(rec.filename,)
)
if existed:
log.warning('%s existed' %(rec.filename, ))
return Result.error(message ='%s existed' %(rec.filename, ))
self.db.execute(
'INSERT INTO ifs_level0_data (level0_id, obs_id, detector_no, obs_type, obs_time, exp_time,detector_status_id, filename, file_path,qc0_status, prc_status,create_time) \
VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?)',
(rec.level0_id, rec.obs_id, rec.detector_no, rec.obs_type, rec.obs_time, rec.exp_time, rec.detector_status_id, rec.filename, rec.file_path,-1,-1,format_time_ms(time.time()))
)
self.db.end()
rec.id = self.db.last_row_id()
file_path = get_parameter(kwargs, "file_path")
copyfiles = get_parameter(kwargs, "copyfiles", False)
try:
rec = ingest_one(file_path, self.db, copyfiles)
return Result.ok_data(data=rec) return Result.ok_data(data=rec)
except Exception as e: except Exception as e:
log.error(e) log.error(e)
return Result.error(message=str(e)) return Result.error(message=str(e))
......
csst_dfs_api_local/mci/ingest.py
View file @ 52824ecc
...@@ -7,6 +7,7 @@ import shutil ...@@ -7,6 +7,7 @@ import shutil
from csst_dfs_api_local.common.db import DBClient from csst_dfs_api_local.common.db import DBClient
from csst_dfs_commons.utils.fits import get_header_value from csst_dfs_commons.utils.fits import get_header_value
from csst_dfs_commons.models.mci import Level0Record
log = logging.getLogger('csst-dfs-api-local') log = logging.getLogger('csst-dfs-api-local')
...@@ -53,7 +54,7 @@ def ingest_one(file_path, db, copyfiles): ...@@ -53,7 +54,7 @@ def ingest_one(file_path, db, copyfiles):
module_id = header["INSTRUME"] module_id = header["INSTRUME"]
obs_type = header["OBSTYPE"] obs_type = header["OBSTYPE"]
object_name = header["OBJECT"] object_name = get_header_value("OBJECT", header, "-")
qc0_status = -1 qc0_status = -1
prc_status = -1 prc_status = -1
...@@ -72,8 +73,8 @@ def ingest_one(file_path, db, copyfiles): ...@@ -72,8 +73,8 @@ def ingest_one(file_path, db, copyfiles):
db.end() db.end()
#level0 #level0
detector = get_header_value("DETNAM", header1, "-") detector = get_header_value("DETNAM", header1, "-")
filename = header["FILENAME"] filename = get_header_value("FILENAME", header, os.path.basename(file_path))
version = get_header_value("IMG_VER", header, "-") version = get_header_value("IMG_VER", header1, "-")
existed = db.exists( existed = db.exists(
"select * from mci_level0_data where filename=?", "select * from mci_level0_data where filename=?",
(filename,) (filename,)
...@@ -116,7 +117,20 @@ def ingest_one(file_path, db, copyfiles): ...@@ -116,7 +117,20 @@ def ingest_one(file_path, db, copyfiles):
db.end() db.end()
rec = Level0Record(
id = level0_id_id,
level0_id = level0_id,
obs_id = obs_id,
detector_no = detector,
obs_type = obs_type,
obs_time = exp_start_time,
exp_time = exp_time,
detector_status_id = detector_status_id,
filename = filename,
file_path = file_full_path
)
print(f"{file_path} imported") print(f"{file_path} imported")
return rec
if __name__ == "__main__": if __name__ == "__main__":
ingest() ingest()
\ No newline at end of file
csst_dfs_api_local/mci/level0.py
View file @ 52824ecc
...@@ -8,6 +8,7 @@ from ..common.utils import * ...@@ -8,6 +8,7 @@ from ..common.utils import *
from csst_dfs_commons.models import Result from csst_dfs_commons.models import Result
from csst_dfs_commons.models.mci import Level0Record from csst_dfs_commons.models.mci import Level0Record
from csst_dfs_commons.models.common import from_dict_list from csst_dfs_commons.models.common import from_dict_list
from .ingest import ingest_one
log = logging.getLogger('csst') log = logging.getLogger('csst')
...@@ -186,46 +187,16 @@ class Level0DataApi(object): ...@@ -186,46 +187,16 @@ class Level0DataApi(object):
''' insert a level0 data record into database ''' insert a level0 data record into database
parameter kwargs: parameter kwargs:
obs_id = [str] file_path = [str],
detector_no = [str] copyfiles = [boolean]
obs_type = [str]
obs_time = [str]
exp_time = [int]
detector_status_id = [int]
filename = [str]
file_path = [str]
return: csst_dfs_common.models.Result return: csst_dfs_common.models.Result
''' '''
rec = Level0Record(
obs_id = get_parameter(kwargs, "obs_id"),
detector_no = get_parameter(kwargs, "detector_no"),
obs_type = get_parameter(kwargs, "obs_type"),
obs_time = get_parameter(kwargs, "obs_time"),
exp_time = get_parameter(kwargs, "exp_time"),
detector_status_id = get_parameter(kwargs, "detector_status_id"),
filename = get_parameter(kwargs, "filename"),
file_path = get_parameter(kwargs, "file_path")
)
rec.level0_id = f"{rec.obs_id}{rec.detector_no}"
try:
existed = self.db.exists(
"select * from mci_level0_data where filename=?",
(rec.filename,)
)
if existed:
log.warning('%s existed' %(rec.filename, ))
return Result.error(message ='%s existed' %(rec.filename, ))
self.db.execute(
'INSERT INTO mci_level0_data (level0_id, obs_id, detector_no, obs_type, obs_time, exp_time,detector_status_id, filename, file_path,qc0_status, prc_status,create_time) \
VALUES(?,?,?,?,?,?,?,?,?,?,?,?)',
(rec.level0_id, rec.obs_id, rec.detector_no, rec.obs_type, rec.obs_time, rec.exp_time, rec.detector_status_id, rec.filename, rec.file_path,-1,-1,format_time_ms(time.time()))
)
self.db.end()
rec.id = self.db.last_row_id()
file_path = get_parameter(kwargs, "file_path")
copyfiles = get_parameter(kwargs, "copyfiles", False)
try:
rec = ingest_one(file_path, self.db, copyfiles)
return Result.ok_data(data=rec) return Result.ok_data(data=rec)
except Exception as e: except Exception as e:
log.error(e) log.error(e)
return Result.error(message=str(e)) return Result.error(message=str(e))
......
csst_dfs_api_local/msc/ingest.py
View file @ 52824ecc
...@@ -65,7 +65,7 @@ def ingest_one(file_path, db, copyfiles): ...@@ -65,7 +65,7 @@ def ingest_one(file_path, db, copyfiles):
db.end() db.end()
#level0 #level0
detector = header["DETECTOR"] detector = header["DETECTOR"]
filename = header["FILENAME"] filename = get_header_value("FILENAME", header, os.path.basename(file_path))
existed = db.exists( existed = db.exists(
"select * from t_level0_data where filename=?", "select * from t_level0_data where filename=?",
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment