level1 fiel

csst_dfs_api_cluster/ifs/level1.py

+import os
 import grpc
 import datetime
 
 from csst_dfs_commons.models import Result
 from csst_dfs_commons.models.common import from_proto_model_list
 from csst_dfs_commons.models.ifs import Level1Record
-
+from csst_dfs_commons.models.constants import UPLOAD_CHUNK_SIZE
 from csst_dfs_proto.ifs.level1 import level1_pb2, level1_pb2_grpc
 
 from ..common.service import ServiceProxy
@@ -160,9 +161,19 @@ class Level1DataApi(object):
             prc_time = get_parameter(kwargs, "prc_time", format_datetime(datetime.now())),
             pipeline_id = get_parameter(kwargs, "pipeline_id")
         )
-        req = level1_pb2.WriteLevel1Req(record = rec)
+        def stream(rec):
+            with open(rec.file_path, 'rb') as f:
+                while True:
+                    data = f.read(UPLOAD_CHUNK_SIZE)
+                    if not data:
+                        break
+                    yield level1_pb2.WriteLevel1Req(record = rec, data = data)
+
         try:
-            resp,_ = self.stub.Write.with_call(req,metadata = get_auth_headers())
+            if not os.path.exists(rec.file_path):
+                return Result.error(message="the file [%s] not existed" % (rec.file_path, ))
+
+            resp,_ = self.stub.Write.with_call(stream(rec),metadata = get_auth_headers())
             if resp.success:
                 return Result.ok_data(data=Level1Record().from_proto_model(resp.record))
             else:

csst_dfs_api_cluster/msc/level1.py

+import os
 import grpc
 import datetime
 
 from csst_dfs_commons.models import Result
 from csst_dfs_commons.models.common import from_proto_model_list
 from csst_dfs_commons.models.msc import Level1Record
-
+from csst_dfs_commons.models.constants import UPLOAD_CHUNK_SIZE
 from csst_dfs_proto.msc.level1 import level1_pb2, level1_pb2_grpc
 
 from ..common.service import ServiceProxy
@@ -154,9 +155,17 @@ class Level1DataApi(object):
             prc_time = get_parameter(kwargs, "prc_time", format_datetime(datetime.now())),
             pipeline_id = get_parameter(kwargs, "pipeline_id")
         )
-        req = level1_pb2.WriteLevel1Req(record = rec)
+        def stream(rec):
+            with open(rec.file_path, 'rb') as f:
+                while True:
+                    data = f.read(UPLOAD_CHUNK_SIZE)
+                    if not data:
+                        break
+                    yield level1_pb2.WriteLevel1Req(record = rec, data = data)
         try:
-            resp,_ = self.stub.Write.with_call(req,metadata = get_auth_headers())
+            if not os.path.exists(rec.file_path):
+                return Result.error(message="the file [%s] not existed" % (rec.file_path, ))
+            resp,_ = self.stub.Write.with_call(stream(rec),metadata = get_auth_headers())
             if resp.success:
                 return Result.ok_data(data=Level1Record().from_proto_model(resp.record))
             else:

tests/test_facility_cal_merge.py

@@ -16,7 +16,7 @@ class CalMergeApiTestCase(unittest.TestCase):
         print('find:', recs)
 
     def test_get_latest_by_l0(self):
-        rec = self.api.get_latest_by_l0(level0_id='000001102', ref_type = "bias")
+        rec = self.api.get_latest_by_l0(level0_id='00000002401', ref_type = "bias")
         print('get_latest_by_l0:', rec)
         
     def test_get(self):
@@ -33,8 +33,8 @@ class CalMergeApiTestCase(unittest.TestCase):
 
     def test_write(self):
         rec = self.api.write(
-            cal_id = "10000X01",
-            detector_no='01', 
+            cal_id = "10000X03",
+            detector_no='03', 
             ref_type = "bias",
             obs_time = "2021-06-04 11:12:13",
             exp_time = 150,

tests/test_facility_detector.py

@@ -12,26 +12,26 @@ class DetectorApiTestCase(unittest.TestCase):
         print('find:', recs)
 
     def test_get(self):
-        rec = self.api.get(no = 'CCD01')
+        rec = self.api.get(no = '02')
         print('get:', rec)
 
     def test_write(self):
-        rec = self.api.write(no = 'CCD02', 
-                detector_name = 'CCD02', 
+        rec = self.api.write(no = '05', 
+                detector_name = 'CCD05', 
                 module_id = 'MSC', 
-                filter_id='f2')
+                filter_id='f5')
         print('write:', rec)
 
     def test_update(self):
-        rec = self.api.update(no = 'CCD01', filter_id = 'f1')
+        rec = self.api.update(no = '01', filter_id = 'f1')
         print('update:', rec)
 
     def test_delete(self):
-        rec = self.api.delete(no = 'CCD01')
+        rec = self.api.delete(no = '01')
         print('delete:', rec)
 
     def test_find_status(self):
-        recs = self.api.find_status(detector_no = 'CCD01', 
+        recs = self.api.find_status(detector_no = '01', 
             status_occur_time = ('2021-06-02','2021-06-08'), 
             limit = 0)
         print('find status:', recs)
@@ -41,5 +41,5 @@ class DetectorApiTestCase(unittest.TestCase):
         print('get status:', rec)
 
     def test_write_status(self):
-        rec = self.api.write_status(detector_no = 'CCD01', status = '{........}',status_time='2021-06-05 12:12:13')
+        rec = self.api.write_status(detector_no = '01', status = '{........}',status_time='2021-06-05 12:12:13')
         print('write status:', rec)
\ No newline at end of file

tests/test_ifs_level1.py

@@ -41,7 +41,7 @@ class IFSLevel1DataTestCase(unittest.TestCase):
             sky_id = 6,            
             prc_status = 3,
             prc_time = '2021-06-05 11:12:13',
-            filename = "dddasd",
-            file_path = "/opt/dddasd.fits",
+            filename = "MSC_MS_210525121500_100000001_09_IFS",
+            file_path = "/opt/temp/csst/MSC_MS_210525121500_100000001_09_raw.fits",
             pipeline_id = "P2")
         print('write:', rec)
\ No newline at end of file

tests/test_msc_level1.py

@@ -38,7 +38,7 @@ class MSCLevel1DataTestCase(unittest.TestCase):
             bias_id = 3,
             prc_status = 3,
             prc_time = '2021-06-04 11:12:13',
-            filename = "dddasd",
-            file_path = "/opt/dddasd.fits",
+            filename = "MSC_MS_210525121500_100000001_09_raw",
+            file_path = "/opt/temp/csst/MSC_MS_210525121500_100000001_09_raw.fits",
             pipeline_id = "P1")
         print('write:', rec)
\ No newline at end of file