From fdb6e5b12a4b89efbf0612861711ff30d81659ad Mon Sep 17 00:00:00 2001 From: shoulinwei Date: Thu, 10 Jun 2021 22:04:38 +0800 Subject: [PATCH] docs --- csst_dfs_api/common/catalog.py | 46 ++++++++-------- csst_dfs_api/facility/calmerge.py | 45 ++++++++-------- csst_dfs_api/facility/detector.py | 48 ++++++++--------- csst_dfs_api/facility/level0.py | 30 +++++------ csst_dfs_api/facility/level0prc.py | 18 +++---- csst_dfs_api/facility/observation.py | 30 +++++------ csst_dfs_api/ifs/level1.py | 30 +++++------ csst_dfs_api/msc/level1.py | 30 +++++------ examples/ifs_rss_pipeline.py | 78 ++++++++++++---------------- 9 files changed, 173 insertions(+), 182 deletions(-) diff --git a/csst_dfs_api/common/catalog.py b/csst_dfs_api/common/catalog.py index 5386e31..731cadd 100644 --- a/csst_dfs_api/common/catalog.py +++ b/csst_dfs_api/common/catalog.py @@ -6,17 +6,18 @@ class CatalogApi(object): self.stub = getattr(self.module, "CatalogApi")() def catalog_query(self, ra: float, dec: float, radius: float, catalog_name: str, min_mag: float, max_mag: float, obstime: int, limit: int): - ''' retrieval catalot - args: - ra: in deg - dec: in deg - radius: in deg - catalog_name: one of ['gaia3','',''] - min_mag: minimal magnitude - max_mag: maximal magnitude - obstime: seconds - limit: limits returns the number of records - return: csst_dfs_common.models.Result + ''' retrieval catalog + + :param ra: in deg + :param dec: in deg + :param radius: in deg + :param catalog_name: one of ['gaia3','',''] + :param min_mag: minimal magnitude + :param max_mag: maximal magnitude + :param obstime: seconds + :param limit: limits returns the number of records + + :returns: csst_dfs_common.models.Result ''' if catalog_name == "gaia3": @@ -25,17 +26,18 @@ class CatalogApi(object): return Result.error(message="%s catalog search not yet implemented" %(catalog_name, )) def gaia3_query(self, ra: float, dec: float, radius: float, min_mag: float, max_mag: float, obstime: int, limit: int): - ''' retrieval GAIA EDR 3 - args: - ra: in deg - dec: in deg - radius: in deg - min_mag: minimal magnitude - max_mag: maximal magnitude - obstime: seconds - limit: limits returns the number of records - return: csst_dfs_common.models.Result - ''' + """retrieval GAIA EDR 3 + + :param ra: in deg + :param dec: in deg + :param radius: in deg + :param min_mag: minimal magnitude + :param max_mag: maximal magnitude + :param obstime: seconds + :param limit: limits returns the number of records + + :returns: csst_dfs_common.models.Result + """ try: return self.stub.gaia3_query(ra, dec, radius, min_mag, max_mag, obstime, limit) except Exception as e: diff --git a/csst_dfs_api/facility/calmerge.py b/csst_dfs_api/facility/calmerge.py index 621023e..fa794ae 100644 --- a/csst_dfs_api/facility/calmerge.py +++ b/csst_dfs_api/facility/calmerge.py @@ -11,55 +11,56 @@ class CalMergeApi(object): def find(self, **kwargs): ''' retrieve calibration merge records from database - - Args: - detector_no: [str] - ref_type: [str] - obs_time: (start,end) - qc1_status : [int] - prc_status : [int] - file_name: [str] + + :param kwargs: Parameter dictionary, key items support: + detector_no: [str], + ref_type: [str], + obs_time: (start,end), + qc1_status : [int], + prc_status : [int], + file_name: [str], limit: limits returns the number of records,default 0:no-limit - Returns: - csst_dfs_common.models.Result + + :returns: csst_dfs_common.models.Result ''' return self.stub.find(**kwargs) def get(self, **kwargs): ''' fetch a record from database - Args: + :param kwargs: Parameter dictionary, key items support: id : [int] - Returns: - csst_dfs_common.models.Result + + :returns: csst_dfs_common.models.Result ''' return self.stub.get(**kwargs) def update_proc_status(self, **kwargs): ''' update the status of reduction - Args: + :param kwargs: Parameter dictionary, key items support: id : [int], status : [int] - Returns: - csst_dfs_common.models.Result + + :returns: csst_dfs_common.models.Result ''' return self.stub.update_proc_status(**kwargs) def update_qc1_status(self, **kwargs): ''' update the status of reduction - AArgs: + :param kwargs: Parameter dictionary, key items support: id : [int], status : [int] - Returns: - csst_dfs_common.models.Result + + :returns: csst_dfs_common.models.Result ''' return self.stub.update_qc1_status(**kwargs) def write(self, **kwargs): ''' insert a calibration merge record into database - Args: + + :param kwargs: Parameter dictionary, key items support: detector_no : [str] ref_type : [str] obs_time : [str] @@ -69,8 +70,8 @@ class CalMergeApi(object): filename : [str] file_path : [str] level0_ids : [list] - Returns: - csst_dfs_common.models.Result + + :returns: csst_dfs_common.models.Result ''' return self.stub.write(**kwargs) diff --git a/csst_dfs_api/facility/detector.py b/csst_dfs_api/facility/detector.py index 48df406..4f9efa5 100644 --- a/csst_dfs_api/facility/detector.py +++ b/csst_dfs_api/facility/detector.py @@ -11,90 +11,90 @@ class DetectorApi(object): def find(self, **kwargs): ''' retrieve detector records from database - Args: + :param kwargs: Parameter dictionary, key items support: module_id: [str] key: [str] - Returns: - csst_dfs_common.models.Result + + :returns: csst_dfs_common.models.Result ''' return self.stub.find(**kwargs) def get(self, **kwargs): ''' fetch a record from database - Args: + :param kwargs: Parameter dictionary, key items support: no : [str] - Returns: - csst_dfs_common.models.Result + + :returns: csst_dfs_common.models.Result ''' return self.stub.get(**kwargs) def update(self, **kwargs): ''' update a detector by no - Args: + :param kwargs: Parameter dictionary, key items support: no : [str], detector_name : [str], module_id : [str], filter_id : [str] - Returns: - csst_dfs_common.models.Result + + :returns: csst_dfs_common.models.Result ''' return self.stub.update(**kwargs) def delete(self, **kwargs): ''' delete a detector by no - Args: + :param kwargs: Parameter dictionary, key items support: no : [str] - Returns: - csst_dfs_common.models.Result + + :returns: csst_dfs_common.models.Result ''' return self.stub.delete(**kwargs) def write(self, **kwargs): ''' insert a detector record into database - Args: + :param kwargs: Parameter dictionary, key items support: no : [str], detector_name : [str], module_id : [str], filter_id : [str] - Returns: - csst_dfs_common.models.Result + + :returns: csst_dfs_common.models.Result ''' return self.stub.write(**kwargs) def find_status(self, **kwargs): ''' retrieve a detector status's from database - Args: + :param kwargs: Parameter dictionary, key items support: detector_no: [str] status_occur_time: (begin,end) limit: limits returns the number of records,default 0:no-limit - Returns: - csst_dfs_common.models.Result + + :returns: csst_dfs_common.models.Result ''' return self.stub.find_status(**kwargs) def get_status(self, **kwargs): ''' fetch a record from database - Args: + :param kwargs: Parameter dictionary, key items support: id : [int] - Returns: - csst_dfs_common.models.Result + + :returns: csst_dfs_common.models.Result ''' return self.stub.get_status(**kwargs) def write_status(self, **kwargs): ''' insert a detector status into database - Args: + :param kwargs: Parameter dictionary, key items support: detector_no : [str], status : [str], status_time : [str] - Returns: - csst_dfs_common.models.Result + + :returns: csst_dfs_common.models.Result ''' return self.stub.write_status(**kwargs) \ No newline at end of file diff --git a/csst_dfs_api/facility/level0.py b/csst_dfs_api/facility/level0.py index 3aea5ee..631452f 100644 --- a/csst_dfs_api/facility/level0.py +++ b/csst_dfs_api/facility/level0.py @@ -12,7 +12,7 @@ class Level0DataApi(object): def find(self, **kwargs): ''' retrieve level0 records from database - Args: + :param kwargs: Parameter dictionary, key items support: obs_id: [int] detector_no: [str] obs_type: [str] @@ -21,47 +21,47 @@ class Level0DataApi(object): prc_status : [int], file_name: [str] limit: limits returns the number of records,default 0:no-limit - Returns: - csst_dfs_common.models.Result + + :returns: csst_dfs_common.models.Result ''' return self.stub.find(**kwargs) def get(self, **kwargs): ''' fetch a record from database - Args: + :param kwargs: Parameter dictionary, key items support: fits_id : [int] - Returns: - csst_dfs_common.models.Result + + :returns: csst_dfs_common.models.Result ''' return self.stub.get(**kwargs) def update_proc_status(self, **kwargs): ''' update the status of reduction - Args: + :param kwargs: Parameter dictionary, key items support: fits_id : [int], status : [int] - Returns: - csst_dfs_common.models.Result + + :returns: csst_dfs_common.models.Result ''' return self.stub.update_proc_status(**kwargs) def update_qc0_status(self, **kwargs): ''' update the status of QC0 - Args: + :param kwargs: Parameter dictionary, key items support: fits_id : [int], status : [int] - Returns: - csst_dfs_common.models.Result + + :returns: csst_dfs_common.models.Result ''' return self.stub.update_qc0_status(**kwargs) def write(self, **kwargs): ''' insert a level0 data record into database - Args: + :param kwargs: Parameter dictionary, key items support: obs_id = [int] detector_no = [str] obs_type = [str] @@ -70,8 +70,8 @@ class Level0DataApi(object): detector_status_id = [int] filename = [str] file_path = [str] - Returns: - csst_dfs_common.models.Result + + :returns: csst_dfs_common.models.Result ''' return self.stub.write(**kwargs) diff --git a/csst_dfs_api/facility/level0prc.py b/csst_dfs_api/facility/level0prc.py index 93ed365..7a58ade 100644 --- a/csst_dfs_api/facility/level0prc.py +++ b/csst_dfs_api/facility/level0prc.py @@ -12,31 +12,31 @@ class Level0PrcApi(object): def find(self, **kwargs): ''' retrieve level0 procedure records from database - Args: + :param kwargs: Parameter dictionary, key items support: level0_id: [int] pipeline_id: [str] prc_module: [str] prc_status : [int] - Returns: - csst_dfs_common.models.Result + + :returns: csst_dfs_common.models.Result ''' return self.stub.find(**kwargs) def update_proc_status(self, **kwargs): ''' update the status of reduction - Args: + :param kwargs: Parameter dictionary, key items support: id : [int], status : [int] - Returns: - csst_dfs_common.models.Result + + :returns: csst_dfs_common.models.Result ''' return self.stub.update_proc_status(**kwargs) def write(self, **kwargs): ''' insert a level0 procedure record into database - Args: + :param kwargs: Parameter dictionary, key items support: level0_id : [int] pipeline_id : [str] prc_module : [str] @@ -44,8 +44,8 @@ class Level0PrcApi(object): prc_status : [int] prc_time : [str] file_path : [str] - Returns: - csst_dfs_common.models.Result + + :returns: csst_dfs_common.models.Result ''' return self.stub.write(**kwargs) diff --git a/csst_dfs_api/facility/observation.py b/csst_dfs_api/facility/observation.py index 201ae2a..c69b70c 100644 --- a/csst_dfs_api/facility/observation.py +++ b/csst_dfs_api/facility/observation.py @@ -12,25 +12,25 @@ class ObservationApi(object): def find(self, **kwargs): ''' retrieve exposure records from database - Args: + :param kwargs: Parameter dictionary, key items support: module_id: [str] obs_type: [str] obs_time : (start, end), qc0_status : [int], prc_status : [int], limit: limits returns the number of records,default 0:no-limit - Returns: - csst_dfs_common.models.Result + + :returns: csst_dfs_common.models.Result ''' return self.stub.find(**kwargs) def get(self, **kwargs): ''' fetch a record from database - Args: + :param kwargs: Parameter dictionary, key items support: obs_id = [int] - Returns: - csst_dfs_common.models.Result + + :returns: csst_dfs_common.models.Result ''' return self.stub.get(**kwargs) @@ -38,29 +38,29 @@ class ObservationApi(object): def update_proc_status(self, **kwargs): ''' update the status of reduction - Args: + :param kwargs: Parameter dictionary, key items support: obs_id = [int], status = [int] - Returns: - csst_dfs_common.models.Result + + :returns: csst_dfs_common.models.Result ''' return self.stub.update_proc_status(**kwargs) def update_qc0_status(self, **kwargs): ''' update the status of QC0 - Args: + :param kwargs: Parameter dictionary, key items support: obs_id = [int], status = [int] - Returns: - csst_dfs_common.models.Result + + :returns: csst_dfs_common.models.Result ''' return self.stub.update_qc0_status(**kwargs) def write(self, **kwargs): ''' insert a observational record into database - Args: + :param kwargs: Parameter dictionary, key items support: obs_id = [int] obs_time = [str] exp_time = [float] @@ -68,8 +68,8 @@ class ObservationApi(object): obs_type = [str] facility_status_id = [int] module_status_id = [int] - Returns: - csst_dfs_common.models.Result + + :returns: csst_dfs_common.models.Result ''' return self.stub.write(**kwargs) diff --git a/csst_dfs_api/ifs/level1.py b/csst_dfs_api/ifs/level1.py index 5ebff70..77e7821 100644 --- a/csst_dfs_api/ifs/level1.py +++ b/csst_dfs_api/ifs/level1.py @@ -12,7 +12,7 @@ class Level1DataApi(object): def find(self, **kwargs): ''' retrieve level1 records from database - Args: + :param kwargs: Parameter dictionary, key items support: raw_id: [int] data_type: [str] obs_type: [str] @@ -21,18 +21,18 @@ class Level1DataApi(object): prc_status : [int], filename: [str] limit: limits returns the number of records,default 0:no-limit - Returns: - csst_dfs_common.models.Result + + :returns: csst_dfs_common.models.Result ''' return self.stub.find(**kwargs) def get(self, **kwargs): ''' fetch a record from database - Args: + :param kwargs: Parameter dictionary, key items support: id : [int] - Returns: - csst_dfs_common.models.Result + + :returns: csst_dfs_common.models.Result ''' return self.stub.get(**kwargs) @@ -40,29 +40,29 @@ class Level1DataApi(object): def update_proc_status(self, **kwargs): ''' update the status of reduction - Args: + :param kwargs: Parameter dictionary, key items support: id = [int], status = [int] - Returns: - csst_dfs_common.models.Result + + :returns: csst_dfs_common.models.Result ''' return self.stub.update_proc_status(**kwargs) def update_qc1_status(self, **kwargs): ''' update the status of QC0 - Args: + :param kwargs: Parameter dictionary, key items support: id = [int], status = [int] - Returns: - csst_dfs_common.models.Result + + :returns: csst_dfs_common.models.Result ''' return self.stub.update_qc1_status(**kwargs) def write(self, **kwargs): ''' insert a level1 record into database - Args: + :param kwargs: Parameter dictionary, key items support: raw_id : [int] data_type : [str] cor_sci_id : [int] @@ -78,8 +78,8 @@ class Level1DataApi(object): prc_status : [int] prc_time : [str] pipeline_id : [str] - Returns: - csst_dfs_common.models.Result + + :returns: csst_dfs_common.models.Result ''' return self.stub.write(**kwargs) diff --git a/csst_dfs_api/msc/level1.py b/csst_dfs_api/msc/level1.py index 7c1708e..a9d58cc 100644 --- a/csst_dfs_api/msc/level1.py +++ b/csst_dfs_api/msc/level1.py @@ -12,7 +12,7 @@ class Level1DataApi(object): def find(self, **kwargs): ''' retrieve level1 records from database - Args: + :param kwargs: Parameter dictionary, key items support: raw_id: [int] data_type: [str] obs_type: [str] @@ -21,18 +21,18 @@ class Level1DataApi(object): prc_status : [int], filename: [str] limit: limits returns the number of records,default 0:no-limit - Returns: - csst_dfs_common.models.Result + + :returns: csst_dfs_common.models.Result ''' return self.stub.find(**kwargs) def get(self, **kwargs): ''' fetch a record from database - Args: + :param kwargs: Parameter dictionary, key items support: id : [int] - Returns: - csst_dfs_common.models.Result + + :returns: csst_dfs_common.models.Result ''' return self.stub.get(**kwargs) @@ -40,29 +40,29 @@ class Level1DataApi(object): def update_proc_status(self, **kwargs): ''' update the status of reduction - Args: + :param kwargs: Parameter dictionary, key items support: id = [int], status = [int] - Returns: - csst_dfs_common.models.Result + + :returns: csst_dfs_common.models.Result ''' return self.stub.update_proc_status(**kwargs) def update_qc1_status(self, **kwargs): ''' update the status of QC0 - Args: + :param kwargs: Parameter dictionary, key items support: id = [int], status = [int] - Returns: - csst_dfs_common.models.Result + + :returns: csst_dfs_common.models.Result ''' return self.stub.update_qc1_status(**kwargs) def write(self, **kwargs): ''' insert a level1 record into database - Args: + :param kwargs: Parameter dictionary, key items support: raw_id : [int] data_type : [str] cor_sci_id : [int] @@ -75,8 +75,8 @@ class Level1DataApi(object): prc_status : [int] prc_time : [str] pipeline_id : [str] - Returns: - csst_dfs_common.models.Result + + :returns: csst_dfs_common.models.Result ''' return self.stub.write(**kwargs) diff --git a/examples/ifs_rss_pipeline.py b/examples/ifs_rss_pipeline.py index 4aac554..16e668f 100644 --- a/examples/ifs_rss_pipeline.py +++ b/examples/ifs_rss_pipeline.py @@ -5,8 +5,8 @@ import logging from astropy.io import fits from csst_dfs_commons.logging import setup_logging -from csst_dfs_api.ifs import FitsApi, RefFitsApi, Result0Api, Result1Api - +from csst_dfs_api.facility import Level0DataApi, CalMergeApi +from csst_dfs_api.ifs import Level1DataApi setup_logging() log = logging.getLogger('csst') @@ -15,16 +15,13 @@ class RSS(object): def __init__(self, file_name): self.root_dir = os.getenv("CSST_LOCAL_FILE_ROOT", "/opt/temp/csst") - self.fitsApi = FitsApi() - self.refFitsApi = RefFitsApi() - self.result0Api = Result0Api() - self.result1Api = Result1Api() + self.level0Api = Level0DataApi() + self.calibrationApi = CalMergeApi() + self.level1Api = Level1DataApi() try: - self.raw = self.fitsApi.find(file_name=file_name) - - if self.raw.success: - self.raw = self.raw.data()[0] if len(self.raw.data())>0 else None + self.raw = self.level0Api.find(file_name=file_name) + self.raw = self.raw.data[0] if self.raw.success else None if self.raw is None: log.error('raw %s not found' %(file_name,)) @@ -36,8 +33,8 @@ class RSS(object): def set_bias(self, file_name=None): try: - self.bias = self.refFitsApi.find(file_name=file_name, ref_type=RefFitsApi.REF_FITS_BIAS) - self.bias = self.bias[0] if self.bias else None + self.bias = self.calibrationApi.find(file_name=file_name, ref_type="bias") + self.bias = self.bias.data[0] if self.bias.success else None if self.bias is None: log.error('bias %s not found' %(file_name,)) @@ -48,8 +45,8 @@ class RSS(object): def set_flat(self, file_name=None): try: - self.flat = self.refFitsApi.find(file_name=file_name, ref_type=RefFitsApi.REF_FITS_FLAT) - self.flat = self.flat[0] if self.flat else None + self.flat = self.calibrationApi.find(file_name=file_name, ref_type="flat") + self.flat = self.flat.data[0] if self.flat.success else None if self.flat is None: log.error('flat %s not found' %(file_name,)) @@ -60,8 +57,8 @@ class RSS(object): def set_arc(self, file_name = None): try: - self.arc = self.refFitsApi.find(file_name=file_name, ref_type=RefFitsApi.REF_FITS_ARC) - self.arc = self.arc[0] if self.arc else None + self.arc = self.calibrationApi.find(file_name=file_name, ref_type="arc") + self.arc = self.arc.data[0] if self.arc.success else None if self.arc is None: log.error('arc %s not found' %(file_name,)) @@ -72,8 +69,8 @@ class RSS(object): def set_sky(self, file_name = None): try: - self.sky = self.refFitsApi.find(file_name=file_name, ref_type=RefFitsApi.REF_FITS_SKY) - self.sky = self.sky[0] if self.sky else None + self.sky = self.calibrationApi.find(file_name=file_name, ref_type="sky") + self.sky = self.sky.data[0] if self.sky.success else None if self.sky is None: log.error('sky %s not found' %(file_name,)) @@ -83,7 +80,6 @@ class RSS(object): log.error('sky %s not found' %(file_name,),e) def makecube(self, outfile): - if self.raw is None: log.error('raw not found') return @@ -97,31 +93,10 @@ class RSS(object): log.error('sky not found') return - hdul_raw = fits.open(os.path.join(self.root_dir, self.raw['file_path'])) - hdul_arc = fits.open(os.path.join(self.root_dir, self.arc['file_path'])) - hdul_flat = fits.open(os.path.join(self.root_dir, self.flat['file_path'])) - hdul_sky = fits.open(os.path.join(self.root_dir, self.sky['file_path'])) - - hdul_raw.append(hdul_arc[0]) - hdul_raw.append(hdul_flat[0]) - hdul_raw.append(hdul_sky[0]) - - hdul_raw.writeto(outfile, overwrite=True) - - self.result0Api.write(raw_id = self.raw['id'], file_path = outfile, proc_type = 'default') - - def makecube2(self, outfile): - refiles = [self.raw, self.arc, self.flat, self.bias, self.sky] - - raw_segments = self.fitsApi.read(self.raw['id']) - arc_segments = self.refFitsApi.read(self.arc['id']) - flat_segments = self.refFitsApi.read(self.flat['id']) - sky_segments = self.refFitsApi.read(self.sky['id']) - - hdul_raw = fits.HDUList.fromstring(b''.join(raw_segments)) - hdul_arc = fits.HDUList.fromstring(b''.join(arc_segments)) - hdul_flat = fits.HDUList.fromstring(b''.join(flat_segments)) - hdul_sky = fits.HDUList.fromstring(b''.join(sky_segments)) + hdul_raw = fits.open(os.path.join(self.raw.file_path)) + hdul_arc = fits.open(os.path.join(self.arc.file_path)) + hdul_flat = fits.open(os.path.join(self.flat.file_path)) + hdul_sky = fits.open(os.path.join(self.sky.file_path)) hdul_raw.append(hdul_arc[0]) hdul_raw.append(hdul_flat[0]) @@ -129,7 +104,20 @@ class RSS(object): hdul_raw.writeto(outfile, overwrite=True) - self.result0Api.write(raw_id = self.raw['id'], file_path = outfile, proc_type = 'default') + self.level1Api.write(level0_id = self.raw.id, + data_type = "sci", + cor_sci_id = 2, + prc_params = "/opt/dddasd.params", + flat_id = self.flat.id, + dark_id = -1, + bias_id = -1, + lamp_id = -1, + arc_id = self.arc.id, + sky_id = self.sky.id, + prc_status = 0, + filename = "rss_demo1", + file_path = outfile, + pipeline_id = "P2") if __name__ == '__main__': rss1 = RSS('CCD1_ObsTime_300_ObsNum_1.fits') # raw data -- GitLab