init

csst_dfs_api_cluster/ifs/__init__.py

+from .fits import FitsApi
+from .reffits import RefFitsApi
+from .result0 import Result0Api
+from .result1 import Result1Api
\ No newline at end of file

csst_dfs_api_cluster/ifs/fits.py

+import os
+import grpc
+
+from csst_dfs_proto.ifs.fits import fits_pb2, fits_pb2_grpc
+
+from ..common.client_config import ClientConfigurator
+from ..common.service import ServiceProxy
+from ..common.utils import *
+from ..common.constants import UPLOAD_CHUNK_SIZE
+
+class FitsApi(object):
+    def __init__(self, sub_system = "ifs"):
+        self.sub_system = sub_system
+        self.proxy = ServiceProxy(ClientConfigurator().gatewayCfg)
+        self.stub = self.proxy.insecure(fits_pb2_grpc.FitsSrvStub)
+
+    def find(self, **kwargs):
+        '''
+        parameter kwargs:
+            obs_time = [int],
+            file_name = [str],
+            exp_time = (start, end),
+            ccd_num = [int],
+            qc0_status = [int],
+            prc_status = [int]
+
+        return list of raw records
+        '''
+        return self.stub.Find(fits_pb2.FindRawFitsReq(
+            obs_time = get_parameter(kwargs, "obs_time"),
+            file_name = get_parameter(kwargs, "file_name"),
+            exp_time_start = get_parameter(kwargs, "exp_time", [None, None])[0],
+            exp_time_end = get_parameter(kwargs, "exp_time", [None, None])[1],
+            ccd_num = get_parameter(kwargs, "ccd_num"),
+            qc0_status = get_parameter(kwargs, "qc0_status"),
+            prc_status = get_parameter(kwargs, "prc_status"),
+            other_conditions = {"test":"cnlab.test"}
+        ))
+
+    def get(self, **kwargs):
+        '''  query database, return a record as dict
+
+        parameter kwargs:
+            fits_id = [int] 
+
+        return dict or None
+        '''
+        return self.stub.Get(fits_pb2.GetRawFitsReq(
+            fits_id = get_parameter(kwargs, "fits_id")
+        ))
+
+    def read(self, **kwargs):
+        ''' yield bytes of fits file
+
+        parameter kwargs:
+            fits_id = [int],
+            file_path = [str], 
+            chunk_size = [int] default 20480
+
+        yield bytes of fits file
+        '''
+        try:
+            streams = self.stub.Read(fits_pb2.ReadRawFitsReq(
+                fits_id = get_parameter(kwargs, "fits_id"),
+                file_path = get_parameter(kwargs, "file_path"),
+                chunk_size = get_parameter(kwargs, "chunk_size", 20480)
+            ))
+
+            for stream in streams:
+                yield stream.data
+
+        except Exception as e:
+            print(e)
+
+    def update_proc_status(self, **kwargs):
+        ''' update the status of reduction
+
+        parameter kwargs:
+            fits_id = [int],
+            status = [int]
+        '''
+        fits_id = get_parameter(kwargs, "fits_id")
+        status = get_parameter(kwargs, "status"),
+        try:
+            return self.stub.update_qc0_status((fits_pb2.UpdateProcStatusReq(fits_id=fits_id, status=status)))
+
+        except grpc.RpcError as identifier:
+            print(identifier)    
+
+    def update_qc0_status(self, **kwargs):
+        ''' update the status of reduction
+        
+        parameter kwargs:
+            fits_id = [int],
+            status = [int]
+        '''        
+        fits_id = get_parameter(kwargs, "fits_id")
+        status = get_parameter(kwargs, "status"),
+        try:
+            return self.stub.update_qc0_status((fits_pb2.UpdateQc0StatusReq(fits_id=fits_id, status=status)))
+
+        except grpc.RpcError as identifier:
+            print(identifier)         
+
+
+    def write(self, **kwargs):
+        ''' copy a local file to file storage, then reduce the header of fits file and insert a record into database
+ 
+        parameter kwargs:
+            file_path = [str]
+        '''   
+        file_path = get_parameter(kwargs, "file_path")
+
+        if os.path.exists(file_path):
+            raise Exception("%s file not found" % (file_path))
+
+        def stream(v_file_path):
+            v_file_name = os.path.basename(v_file_path)
+            with open(v_file_path, 'rb') as f:
+                while True:
+                    data = f.read(UPLOAD_CHUNK_SIZE)
+                    if not data:
+                        break
+                    yield fits_pb2.WriteRawFitsReq(
+                                    file_name=v_file_name, 
+                                    fitsData=data)
+
+        return self.stub.Write(stream(file_path))
+    
+
+   
+

csst_dfs_api_cluster/ifs/reffits.py

+
+import os
+import grpc
+
+from ..common.client_config import ClientConfigurator
+from ..common.service import ServiceProxy
+from ..common.utils import *
+from ..common.constants import *
+import grpc
+
+class RefFitsApi(object):
+    def __init__(self, sub_system):
+        self.sub_system = sub_system
+
+

csst_dfs_api_cluster/ifs/result0.py

+
+import os
+import grpc
+
+from ..common.client_config import ClientConfigurator
+from ..common.service import ServiceProxy
+from ..common.utils import *
+from ..common.constants import *
+import grpc
+
+
+class Result0Api(object):
+    def __init__(self, sub_system):
+        self.sub_system = sub_system
+
+

csst_dfs_api_cluster/ifs/result1.py

+
+import os
+import grpc
+
+from ..common.client_config import ClientConfigurator
+from ..common.service import ServiceProxy
+from ..common.utils import *
+from ..common.constants import *
+import grpc
+
+class Result1Api(object):
+    def __init__(self, sub_system = "ifs"):
+        self.sub_system = sub_system
+
+

requirements.txt

+astropy
+grpc
+orjson
+git+https://github.com/astronomical-data-processing/csst-dfs-proto-py.git
\ No newline at end of file

setup.cfg

+[metadata]
+# replace with your username:
+name = csst_dfs_api_cluster
+author =CSST DFS Team.
+author_email = weishoulin@astrolab.cn
+description = CSST DFS Cluster APIs Library.
+long_description = file: README.md
+long_description_content_type = text/markdown
+keywords = astronomy, astrophysics, cosmology, space, CSST
+url = https://github.com/astronomical-data-processing/csst-dfs-api-cluster
+project_urls =
+    Bug Tracker = https://github.com/astronomical-data-processing/csst-dfs-api-cluster/issues
+classifiers =
+    Programming Language :: Python :: 3
+    License :: OSI Approved :: MIT License
+    Operating System :: OS Independent
+
+[options]
+packages = find:
+python_requires = >=3.7
+zip_safe = False
+setup_requires = setuptools_scm
+install_requires =
+    astropy>=4.0

setup.py

+# coding: utf-8
+import os
+from setuptools import setup
+
+setup(use_scm_version={'write_to': os.path.join('csst_dfs_api_cluster', '_version.py')})

tests/test_common_catalog.py

+import os
+import unittest
+from astropy.io import fits
+
+from csst_dfs_api_cluster.common.catalog import CatalogApi
+
+class CommonEphemTestCase(unittest.TestCase):
+
+    def setUp(self):
+        self.api = CatalogApi()
+
+    def test_gaia3_query(self):
+        recs = self.api.gaia3_query(ra=56.234039029108935, dec=14.466534827703473, radius=4, min_mag=-1, max_mag=-1, obstime = -1, limit = 2)
+        print('find:', recs)

tests/test_ifs_fits.py

+import os
+import unittest
+from astropy.io import fits
+
+from csst_dfs_api_cluster.ifs import FitsApi
+
+class IFSFitsTestCase(unittest.TestCase):
+
+    def setUp(self):
+        self.api = FitsApi()
+
+    def test_find(self):
+        recs = self.api.find(file_name='CCD1_ObsTime_300_ObsNum_7.fits')
+        print('find:', recs)
+        assert len(recs) == 1
+
+        recs = self.api.find()
+        print('find:', recs)
+        assert len(recs) > 1
+
+    # def test_read(self):
+    #     recs = self.api.find(file_name='CCD1_ObsTime_300_ObsNum_7.fits')
+    #     print("The full path: ", os.path.join(self.api.root_dir, recs[0]['file_path']))
+
+    #     file_segments = self.api.read(file_path=recs[0]['file_path'])
+    #     file_bytes = b''.join(file_segments)
+    #     hdul = fits.HDUList.fromstring(file_bytes)
+    #     print(hdul.info())
+    #     hdr = hdul[0].header
+    #     print(repr(hdr))      
+
+    # def test_update_proc_status(self):
+    #     recs = self.api.find(file_name='CCD1_ObsTime_300_ObsNum_7.fits')
+
+    #     self.api.update_proc_status(fits_id=recs[0]['id'],status=1)
+
+    #     rec = self.api.get(fits_id=recs[0]['id'])
+    #     assert rec['prc_status'] == 1
+
+    # def test_update_qc0_status(self):
+    #     recs = self.api.find(file_name='CCD1_ObsTime_300_ObsNum_7.fits')
+
+    #     self.api.update_qc0_status(fits_id=recs[0]['id'],status=1)
+
+    #     rec = self.api.get(fits_id=recs[0]['id'])
+    #     assert rec['qc0_status'] == 1
+
+    # def test_write(self):
+    #     recs = self.api.write(file_path='/opt/temp/csst_ifs/CCD2_ObsTime_1200_ObsNum_40.fits')
+
+    #     recs = self.api.find(file_name='CCD2_ObsTime_1200_ObsNum_40.fits')
+
+    #     rec = self.api.get(fits_id=recs[0]['id'])
+
+    #     print(rec)
\ No newline at end of file