Commit 7a504935 authored by Wei Shoulin's avatar Wei Shoulin
Browse files

refactor(level2): rename qc2_status to qc_status and update function signatures

parent 10acbf17
Pipeline #11720 failed with stages
in 0 seconds
DEFAULT_DATASET = 'routine' DEFAULT_DATASET = 'routine'
DEFAULT_BATCH_ID = 'auto' DEFAULT_BATCH_ID = 'default'
INSTRUMENTS = ['MSC', 'IFS', 'MCI', 'HSTDM', 'CPIC'] INSTRUMENTS = ['MSC', 'IFS', 'MCI', 'HSTDM', 'CPIC']
\ No newline at end of file
...@@ -6,6 +6,8 @@ from .common import request, Result, utils, constants ...@@ -6,6 +6,8 @@ from .common import request, Result, utils, constants
DateTimeTuple = Tuple[str, str] DateTimeTuple = Tuple[str, str]
def find( def find(
dataset: str,
batch_id: str,
obs_group: Optional[str] = None, obs_group: Optional[str] = None,
obs_id: Optional[str] = None, obs_id: Optional[str] = None,
instrument: str = 'MSC', instrument: str = 'MSC',
...@@ -14,18 +16,19 @@ def find( ...@@ -14,18 +16,19 @@ def find(
filter: Optional[str] = None, filter: Optional[str] = None,
obs_time: Optional[DateTimeTuple] = None, obs_time: Optional[DateTimeTuple] = None,
create_time: Optional[DateTimeTuple] = None, create_time: Optional[DateTimeTuple] = None,
qc2_status: Optional[int] = None, qc_status: Optional[int] = None,
prc_status: Optional[int] = None, prc_status: Optional[int] = None,
file_name: Optional[str] = None, file_name: Optional[str] = None,
object_name: Optional[str] = None, object_name: Optional[str] = None,
dataset: str = constants.DEFAULT_DATASET, brick_id: Optional[int] = None,
batch_id: str = constants.DEFAULT_BATCH_ID,
page: int = 1, page: int = 1,
limit: int = 0) -> Result: limit: int = 0) -> Result:
""" """
根据给定的参数搜索2级数据文件记录 根据给定的参数搜索2级数据文件记录
Args: Args:
dataset (Optional[str], optional): 数据集名称.
batch_id (Optional[str], optional): 批次ID.
obs_group (Optional[str], optional): 项目ID. Defaults to None. obs_group (Optional[str], optional): 项目ID. Defaults to None.
obs_id (Optional[str], optional): 观测ID. Defaults to None. obs_id (Optional[str], optional): 观测ID. Defaults to None.
instrument (Optional[str], optional): 模块ID,如'MSC', 'IFS'. Defaults to None. instrument (Optional[str], optional): 模块ID,如'MSC', 'IFS'. Defaults to None.
...@@ -34,12 +37,11 @@ def find( ...@@ -34,12 +37,11 @@ def find(
filter (Optional[str], optional): 滤光片. Defaults to None. filter (Optional[str], optional): 滤光片. Defaults to None.
obs_time (Optional[DateTimeTuple], optional): 观测时间范围. Defaults to None. obs_time (Optional[DateTimeTuple], optional): 观测时间范围. Defaults to None.
create_time (Optional[DateTimeTuple], optional): 创建时间范围. Defaults to None. create_time (Optional[DateTimeTuple], optional): 创建时间范围. Defaults to None.
qc2_status (Optional[int], optional): QC0状态. Defaults to None. qc_status (Optional[int], optional): QC状态. Defaults to None.
prc_status (Optional[int], optional): 处理状态. Defaults to None. prc_status (Optional[int], optional): 处理状态. Defaults to None.
file_name (Optional[str], optional): 文件名. Defaults to None. file_name (Optional[str], optional): 文件名. Defaults to None.
object_name (Optional[str], optional): 天体名称. Defaults to None. object_name (Optional[str], optional): 天体名称. Defaults to None.
dataset (Optional[str], optional): 数据集名称. Defaults to None. brick_id (Optional[int], optional): brick ID. Defaults to None.
batch_id (Optional[str], optional): 批次ID. Defaults to None.
page (int, optional): 页码. Defaults to 1. page (int, optional): 页码. Defaults to 1.
limit (int, optional): 每页数量. Defaults to 0. limit (int, optional): 每页数量. Defaults to 0.
...@@ -56,7 +58,7 @@ def find( ...@@ -56,7 +58,7 @@ def find(
'detector': detector, 'detector': detector,
'data_model': data_model, 'data_model': data_model,
'filter': filter, 'filter': filter,
'qc2_status': qc2_status, 'qc_status': qc_status,
'prc_status': prc_status, 'prc_status': prc_status,
'file_name': file_name, 'file_name': file_name,
'object_name': object_name, 'object_name': object_name,
...@@ -66,6 +68,7 @@ def find( ...@@ -66,6 +68,7 @@ def find(
'obs_time_end': None, 'obs_time_end': None,
'create_time_start': None, 'create_time_start': None,
'create_time_end': None, 'create_time_end': None,
'brick_id': brick_id,
'page': page, 'page': page,
'limit': limit, 'limit': limit,
} }
...@@ -92,19 +95,19 @@ def find_by_level2_id(level2_id: str) -> Result: ...@@ -92,19 +95,19 @@ def find_by_level2_id(level2_id: str) -> Result:
""" """
return request.get(f"/api/level2/{level2_id}") return request.get(f"/api/level2/{level2_id}")
def update_qc2_status(level2_id: str, data_model: str, qc2_status: int, batch_id: str = constants.DEFAULT_BATCH_ID) -> Result: def update_qc_status(level2_id: str, data_model: str, qc_status: int, batch_id: str = constants.DEFAULT_BATCH_ID) -> Result:
""" """
更新2级数据的QC0状态 更新2级数据的QC状态
Args: Args:
level2_id (str): 2级数据文件的ID level2_id (str): 2级数据文件的ID
data_model (str): 数据类型,如'csst-msc-l2-mbi-cat' data_model (str): 数据类型,如'csst-msc-l2-mbi-cat'
qc2_status (int): QC0状态 qc_status (int): QC状态
Returns: Returns:
Result: 更新结果 Result: 更新结果
""" """
return request.put(f"/api/level2/qc2_status/{level2_id}", {'data_model': data_model, 'qc2_status': qc2_status, 'batch_id': batch_id}) return request.put(f"/api/level2/qc_status/{level2_id}", {'data_model': data_model, 'qc_status': qc_status, 'batch_id': batch_id})
def update_prc_status(level2_id: str, data_model: str, prc_status: int, batch_id: str = constants.DEFAULT_BATCH_ID) -> Result: def update_prc_status(level2_id: str, data_model: str, prc_status: int, batch_id: str = constants.DEFAULT_BATCH_ID) -> Result:
""" """
...@@ -120,18 +123,18 @@ def update_prc_status(level2_id: str, data_model: str, prc_status: int, batch_id ...@@ -120,18 +123,18 @@ def update_prc_status(level2_id: str, data_model: str, prc_status: int, batch_id
""" """
return request.put(f"/api/level2/prc_status/{level2_id}", {'data_model': data_model, 'prc_status': prc_status, 'batch_id': batch_id}) return request.put(f"/api/level2/prc_status/{level2_id}", {'data_model': data_model, 'prc_status': prc_status, 'batch_id': batch_id})
def update_qc2_status_by_file_name(file_name: str, qc2_status: int, batch_id: str = constants.DEFAULT_BATCH_ID) -> Result: def update_qc_status_by_file_name(file_name: str, qc_status: int, batch_id: str = constants.DEFAULT_BATCH_ID) -> Result:
""" """
更新2级数据的QC0状态 更新2级数据的QC状态
Args: Args:
file_name (str): 2级数据文件名 file_name (str): 2级数据文件名
qc2_status (int): QC0状态 qc_status (int): QC状态
Returns: Returns:
Result: 更新结果 Result: 更新结果
""" """
return request.put(f"/api/level2/qc2_status/file/{file_name}", {'qc2_status': qc2_status, 'batch_id': batch_id}) return request.put(f"/api/level2/qc_status/file/{file_name}", {'qc_status': qc_status, 'batch_id': batch_id})
def update_prc_status_by_file_name(file_name: str, prc_status: int, batch_id: str = constants.DEFAULT_BATCH_ID) -> Result: def update_prc_status_by_file_name(file_name: str, prc_status: int, batch_id: str = constants.DEFAULT_BATCH_ID) -> Result:
""" """
...@@ -146,59 +149,19 @@ def update_prc_status_by_file_name(file_name: str, prc_status: int, batch_id: st ...@@ -146,59 +149,19 @@ def update_prc_status_by_file_name(file_name: str, prc_status: int, batch_id: st
""" """
return request.put(f"/api/level2/prc_status/file/{file_name}", {'prc_status': prc_status, 'batch_id': batch_id}) return request.put(f"/api/level2/prc_status/file/{file_name}", {'prc_status': prc_status, 'batch_id': batch_id})
def write(local_file: Union[IO, str], def write(local_file: Union[IO, str]) -> Result:
instrument: Literal['MSC', 'IFS', 'MCI', 'HSTDM', 'CPIC'],
level2_id: str,
data_model: str,
file_name: str,
dag: str,
build: int,
level0_id: Optional[str] = None,
level1_id: Optional[str] = None,
brick_id: Optional[int] = 0,
dataset: str = constants.DEFAULT_DATASET,
batch_id: str = constants.DEFAULT_BATCH_ID,
qc2_status: int = 0,
**extra_kwargs) -> Result:
""" """
将本地的2级数据文件写入到DFS中 将本地的2级数据文件写入到DFS中
Args: Args:
local_file (Union[IO, str]): 文件路径 或 文件对象 local_file (Union[IO, str]): 文件路径 或 文件对象
instrument ['MSC', 'IFS', 'MCI', 'HSTDM', 'CPIC']其中一个,代表: 模块ID
level2_id (str): 2级数据的ID
data_model (str): 数据类型,如'csst-msc-l2-mbi-cat'
file_name (str): 2级数据文件名
dag (str): DAG标识
build (int): 构建号
level0_id (Optional[str]): 0级数据的ID默认为 None
level1_id (Optional[str]): 1级数据的ID默认为 None
brick_id (Optional[int]): 天区的ID默认为 0
dataset (Optional[str], optional): 数据集名称. Defaults to None.
batch_id (Optional[str], optional): 最后一次成功的批次ID. Defaults to None.
qc2_status (int): QC0状态默认为 0
**kwargs: 额外的关键字参数,这些参数将传递给DFS
Returns: Returns:
Result: 操作的结果对象,包含操作是否成功以及相关的错误信息,成功返回data为2级数据对象 Result: 操作的结果对象,包含操作是否成功以及相关的错误信息,成功返回data为2级数据对象
""" """
params = { params = {}
'instrument': instrument,
'level0_id': level0_id,
'level1_id': level1_id,
'level2_id': level2_id,
'brick_id': brick_id,
'file_name': file_name,
'data_model': data_model,
'dag': dag,
'build': build,
'dataset': dataset,
'batch_id': batch_id,
'qc2_status': qc2_status
}
params.update(extra_kwargs)
if local_file is None: if local_file is None:
raise ValueError("local_file is required") raise ValueError("local_file is required")
if isinstance(local_file, str): if isinstance(local_file, str):
......
...@@ -19,8 +19,8 @@ class Level2TestCase(unittest.TestCase): ...@@ -19,8 +19,8 @@ class Level2TestCase(unittest.TestCase):
self.assertEqual(result.code, 200, "error code: " + str(result.code) + ", message: " + result.message) self.assertEqual(result.code, 200, "error code: " + str(result.code) + ", message: " + result.message)
self.assertIsNotNone(result.data, "error message: " + result.message) self.assertIsNotNone(result.data, "error message: " + result.message)
def test_update_qc2_status(self): def test_update_qc_status(self):
result = level2.update_qc2_status(level2_id = "1010940063886712", data_model='csst-msc-l2-mbi-cat', qc2_status=1) result = level2.update_qc_status(level2_id = "1010940063886712", data_model='csst-msc-l2-mbi-cat', qc_status=1)
print(result) print(result)
self.assertEqual(result.code, 200, "error code: " + str(result.code) + ", message: " + result.message) self.assertEqual(result.code, 200, "error code: " + str(result.code) + ", message: " + result.message)
...@@ -29,31 +29,19 @@ class Level2TestCase(unittest.TestCase): ...@@ -29,31 +29,19 @@ class Level2TestCase(unittest.TestCase):
print(result) print(result)
self.assertEqual(result.code, 200, "error code: " + str(result.code) + ", message: " + result.message) self.assertEqual(result.code, 200, "error code: " + str(result.code) + ", message: " + result.message)
def test_update_qc2_status_by_file_name(self): def test_update_qc_status_by_file_name(self):
result = level2.update_qc2_status_by_file_name(file_name = "CSST_MSC_MS_SCI_20310423084104_20310423084334_10109400638867_12_L2_V01_CATMIX.fits", qc2_status=1) result = level2.update_qc_status_by_file_name(file_name = "CSST_MSC_MS_WIDE_20230227011213_20230227011443_10100059495_12_L1_V01_CAT.fits", qc_status=1)
print(result) print(result)
self.assertEqual(result.code, 200, "error code: " + str(result.code) + ", message: " + result.message) self.assertEqual(result.code, 200, "error code: " + str(result.code) + ", message: " + result.message)
def test_update_prc_status_by_file_name(self): def test_update_prc_status_by_file_name(self):
result = level2.update_prc_status_by_file_name(file_name = "CSST_MSC_MS_SCI_20310423084104_20310423084334_10109400638867_12_L2_V01_CATMIX.fits", prc_status=3) result = level2.update_prc_status_by_file_name(file_name = "CSST_MSC_MS_WIDE_20230227011213_20230227011443_10100059495_12_L1_V01_CAT.fits", prc_status=3)
print(result) print(result)
self.assertEqual(result.code, 200, "error code: " + str(result.code) + ", message: " + result.message) self.assertEqual(result.code, 200, "error code: " + str(result.code) + ", message: " + result.message)
def test_write(self): def test_write(self):
file_path = "/Users/wsl/temp/csst/import/CSST_MSC_MS_SCI_20310423084104_20310423084334_10109400638867_12_L2_V01_CATMIX.fits" file_path = "/Users/wsl/temp/csst/conf-test/CSST_MSC_MS_WIDE_20230227011213_20230227011443_10100059495_12_L1_V01_CAT.fits"
result = level2.write(local_file = file_path, result = level2.write(local_file = file_path)
instrument = "MSC",
level0_id="1010940063886712",
level1_id = "1010940063886712",
level2_id = "1010940063886712",
data_model='csst-msc-l2-mbi-cat',
file_name = "CSST_MSC_MS_SCI_20310423084104_20310423084334_10109400638867_12_L2_V01_CATMIX.fits",
brick_id = 254,
qc2_status=12,
dag = "csst-msc-l2-mbi",
build=1,
dataset="094",
prc_status=1)
print(result) print(result)
self.assertEqual(result.code, 200, "error code: " + str(result.code) + ", message: " + result.message) self.assertEqual(result.code, 200, "error code: " + str(result.code) + ", message: " + result.message)
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment