Skip to content
GitLab
Commit 65c0fd80 authored by Wei Shoulin's avatar Wei Shoulin
Browse files

reconstruct

parent cb2e0a9e
Hide whitespace changes
Inline Side-by-side
csst_dfs_api_local/common/utils.py
View file @ 65c0fd80
......@@ -54,4 +54,15 @@ def create_dir(root_dir, sub_system, time_str: list):
the_dir = os.path.join(root_dir, sub_system, time_str)
if not os.path.exists(the_dir):
os.makedirs(os.path.join(root_dir, sub_system, time_str))
return the_dir
\ No newline at end of file
return the_dir
def yield_file_bytes(full_file_path, chunk_size):
if not os.path.exists(full_file_path):
raise Exception("%s file not found" %(full_file_path))
with open(full_file_path, 'rb') as f:
while True:
data = f.read(chunk_size)
if not data:
break
yield data
\ No newline at end of file
csst_dfs_api_local/ifs/fits.py
View file @ 65c0fd80
......@@ -11,10 +11,9 @@ from glob import glob
from astropy.io import fits
from ..common.db import DBClient
from ..common.utils import get_parameter, format_time_ms, create_dir
from ..common.utils import *
log = logging.getLogger('csst')
class FitsApi(object):
def __init__(self, sub_system = "ifs"):
self.sub_system = sub_system
......@@ -36,14 +35,14 @@ class FitsApi(object):
def find(self, **kwargs):
'''
parameter kwargs:
obs_time = [int]
file_name = [str]
exp_time = (start, end)
ccd_num = [int]
qc0_status = [int]
obs_time = [int],
file_name = [str],
exp_time = (start, end),
ccd_num = [int],
qc0_status = [int],
prc_status = [int]
return list of paths
return list of raw records
'''
paths = []
......@@ -90,9 +89,9 @@ class FitsApi(object):
def read(self, **kwargs):
'''
parameter kwargs:
fits_id = [int]
file_path = [str]
chunk_size = [int]
fits_id = [int],
file_path = [str],
chunk_size = [int] default 20480
yield bytes of fits file
'''
......@@ -106,23 +105,17 @@ class FitsApi(object):
r = self.db.select_one(
"select * from ifs_rawfits where id=?", (fits_id,))
if r is not None:
file_path = os.path.join(self.root_dir, r["file_path"])
file_path = r["file_path"]
if file_path is not None:
path = os.path.join(self.root_dir, file_path)
chunk_size = get_parameter(kwargs, "chunk_size", 1024)
with open(path, 'rb') as f:
while True:
data = f.read(chunk_size)
if not data:
break
yield data
chunk_size = get_parameter(kwargs, "chunk_size", 20480)
return yield_file_bytes(os.path.join(self.root_dir, file_path), chunk_size)
def update_proc_status(self, **kwargs):
'''
parameter kwargs:
fits_id = [int]
status = [0 or 1]
fits_id = [int],
status = [int]
'''
fits_id = get_parameter(kwargs, "fits_id")
status = get_parameter(kwargs, "status")
......@@ -143,7 +136,8 @@ class FitsApi(object):
def update_qc0_status(self, **kwargs):
'''
parameter kwargs:
fits_id = [int]
fits_id = [int],
status = [int]
'''
fits_id = get_parameter(kwargs, "fits_id")
......@@ -164,10 +158,9 @@ class FitsApi(object):
def import2db(self, **kwargs):
'''
reduce the header of fits file of server and insert a record into database
parameter kwargs:
file_path = [str]
upload to database
'''
file_path = get_parameter(kwargs, "file_path")
......@@ -204,6 +197,11 @@ class FitsApi(object):
log.info("raw fits %s imported.", file_path)
def write(self, **kwargs):
'''
copy a local file to file storage, then reduce the header of fits file and insert a record into database
parameter kwargs:
file_path = [str]
'''
file_path = get_parameter(kwargs, "file_path")
if not file_path:
......
csst_dfs_api_local/ifs/reffits.py
View file @ 65c0fd80
......@@ -5,11 +5,9 @@ import shutil
from astropy.io import fits
from ..common.db import DBClient
from ..common.utils import get_parameter, format_time_ms,create_dir
from . import FitsApi
from ..common.utils import *
log = logging.getLogger('csst')
class RefFitsApi(object):
REF_FITS_BIAS = "bias"
REF_FITS_FLAT = "flat"
......@@ -35,13 +33,13 @@ class RefFitsApi(object):
def find(self, **kwargs):
'''
parameter kwargs:
obs_time = [int]
file_name = [str]
exp_time = (start, end)
status = [int]
obs_time = [int],
file_name = [str],
exp_time = (start, end),
status = [int],
ref_type = [str]
return list of paths
return list of reference's files records
'''
obs_time = get_parameter(kwargs, "obs_time")
file_name = get_parameter(kwargs, "file_name")
......@@ -73,7 +71,8 @@ class RefFitsApi(object):
return r
def get(self, **kwargs):
'''
'''query database, return a record as dict
parameter kwargs:
fits_id = [int]
......@@ -87,8 +86,8 @@ class RefFitsApi(object):
def read(self, **kwargs):
'''
parameter kwargs:
fits_id = [int]
file_path = [str]
fits_id = [int],
file_path = [str],
chunk_size = [int]
yield bytes of fits file
......@@ -103,22 +102,17 @@ class RefFitsApi(object):
r = self.db.select_one(
"select * from ifs_ref_fits where id=?", (fits_id))
if r is not None:
file_path = os.path.join(self.root_dir, r["file_path"])
file_path = r["file_path"]
if file_path is not None:
path = os.path.join(self.root_dir, file_path)
chunk_size = get_parameter(kwargs, "chunk_size", 1024)
with open(path, 'rb') as f:
while True:
data = f.read(chunk_size)
if not data:
break
yield data
chunk_size = get_parameter(kwargs, "chunk_size", 20480)
return yield_file_bytes(os.path.join(self.root_dir, file_path), chunk_size)
def update_status(self, **kwargs):
'''
parameter kwargs:
fits_id = [int]
fits_id = [int],
status = [int]
'''
fits_id = get_parameter(kwargs, "fits_id")
......@@ -197,6 +191,11 @@ class RefFitsApi(object):
log.info("ref fits %s imported.", file_path)
def write(self, **kwargs):
''' copy a local file to file storage, then reduce the header of fits file and insert a record into database
parameter kwargs:
file_path = [str]
'''
file_path = get_parameter(kwargs, "file_path")
new_file_dir = create_dir(os.path.join(self.root_dir, "refs"),
......
csst_dfs_api_local/ifs/result0.py
View file @ 65c0fd80
......@@ -4,7 +4,7 @@ import time, datetime
import shutil
from ..common.db import DBClient
from ..common.utils import get_parameter, format_time_ms, create_dir
from ..common.utils import *
log = logging.getLogger('csst')
......@@ -26,11 +26,11 @@ class Result0Api(object):
def find(self, **kwargs):
'''
parameter kwargs:
raw_id = [int]
file_name = [str]
raw_id = [int],
file_name = [str],
proc_type = [str]
return list of paths
return list of level 0 record
'''
paths = []
......@@ -52,7 +52,8 @@ class Result0Api(object):
return r
def get(self, **kwargs):
'''
''' query database, return a record as dict
parameter kwargs:
fits_id = [int]
......@@ -64,11 +65,12 @@ class Result0Api(object):
return r
def read(self, **kwargs):
'''
''' yield bytes of fits file
parameter kwargs:
fits_id = [int]
file_path = [str]
chunk_size = [int]
fits_id = [int],
file_path = [str],
chunk_size = [int] default 20480
yield bytes of fits file
'''
......@@ -82,28 +84,20 @@ class Result0Api(object):
r = self.db.select_one(
"select * from ifs_result_0 where id=?", (fits_id))
if r is not None:
file_path = os.path.join(self.root_dir, r["file_path"])
file_path = r["file_path"]
if file_path is not None:
path = os.path.join(self.root_dir, file_path)
chunk_size = get_parameter(kwargs, "chunk_size", 1024)
with open(path, 'rb') as f:
while True:
data = f.read(chunk_size)
if not data:
break
yield data
chunk_size = get_parameter(kwargs, "chunk_size", 20480)
return yield_file_bytes(os.path.join(self.root_dir, file_path), chunk_size)
def write(self, **kwargs):
'''
''' copy a local level 0 file to file storage, and insert a record into database
parameter kwargs:
raw_id = [int]
file_path = [str]
raw_id = [int],
file_path = [str],
proc_type = [str]
insert into database
'''
raw_id = get_parameter(kwargs, "raw_id")
file_path = get_parameter(kwargs, "file_path")
proc_type = get_parameter(kwargs, "proc_type", "default")
......
csst_dfs_api_local/ifs/result1.py
View file @ 65c0fd80
......@@ -4,10 +4,9 @@ import time, datetime
import shutil
from ..common.db import DBClient
from ..common.utils import get_parameter, format_time_ms, create_dir
from ..common.utils import *
log = logging.getLogger('csst')
class Result1Api(object):
def __init__(self, sub_system = "ifs"):
self.sub_system = sub_system
......@@ -26,10 +25,10 @@ class Result1Api(object):
def find(self, **kwargs):
'''
parameter kwargs:
file_name = [str]
file_name = [str],
proc_type = [str]
return list of paths
return list of level 1 record
'''
paths = []
......@@ -61,11 +60,12 @@ class Result1Api(object):
return r, result0s
def read(self, **kwargs):
'''
''' yield bytes of fits file
parameter kwargs:
fits_id = [int]
file_path = [str]
chunk_size = [int]
fits_id = [int],
file_path = [str],
chunk_size = [int] default 20480
yield bytes of fits file
'''
......@@ -79,23 +79,17 @@ class Result1Api(object):
r = self.db.select_one(
"select * from ifs_result_1 where id=?", (fits_id))
if r is not None:
file_path = os.path.join(self.root_dir, r["file_path"])
file_path = r["file_path"]
if file_path is not None:
path = os.path.join(self.root_dir, file_path)
chunk_size = get_parameter(kwargs, "chunk_size", 1024)
with open(path, 'rb') as f:
while True:
data = f.read(chunk_size)
if not data:
break
yield data
chunk_size = get_parameter(kwargs, "chunk_size", 20480)
return yield_file_bytes(os.path.join(self.root_dir, file_path), chunk_size)
def write(self, **kwargs):
'''
''' copy a local level 1 file to file storage, and insert a record into database
parameter kwargs:
file_path = [str]
proc_type = [str]
file_path = [str],
proc_type = [str],
result0_ids = [list]
insert into database
......
tests/test_ifs_fits.py
View file @ 65c0fd80
import os
import unittest
from astropy.io import fits
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment