level0.py 9.82 KB
Newer Older
Wei Shoulin's avatar
Wei Shoulin committed
1
import os
Wei Shoulin's avatar
t    
Wei Shoulin committed
2
from typing import Optional, Tuple, Literal
Wei Shoulin's avatar
Wei Shoulin committed
3
from .common import request, Result, utils, constants
Wei Shoulin's avatar
Wei Shoulin committed
4
5
6

DateTimeTuple = Tuple[str, str]

7
8
9
def find(
        instrument: Literal['MSC', 'IFS', 'MCI', 'HSTDM', 'CPIC'], 
        obs_group: Optional[str] = None,
Wei Shoulin's avatar
Wei Shoulin committed
10
        obs_id: Optional[str] = None,
11
        detector: Optional[str] = None,
12
        obs_type: Optional[str] = None,
Wei Shoulin's avatar
Wei Shoulin committed
13
14
15
16
17
18
19
20
21
22
        filter: Optional[str] = None,
        obs_time: Optional[DateTimeTuple] = None,
        create_time: Optional[DateTimeTuple] = None,
        qc0_status: Optional[int] = None,
        prc_status: Optional[int] = None,
        file_name: Optional[str] = None,
        ra_obj: Optional[int] = None,
        dec_obj: Optional[int] = None,
        radius: Optional[float] = None,
        object_name: Optional[str] = None,
Wei Shoulin's avatar
Wei Shoulin committed
23
        dataset: str = constants.DEFAULT_DATASET,
Wei Shoulin's avatar
Wei Shoulin committed
24
25
26
        page: int = 1,
        limit: int = 0) -> Result:
    """
Wei Shoulin's avatar
Wei Shoulin committed
27
    根据给定的参数搜索0级数据文件记录
Wei Shoulin's avatar
Wei Shoulin committed
28
29
    
    Args:
30
        instrument (str): 设备,必需为'MSC', 'IFS', 'MCI', 'HSTDM', 'CPIC'之一.
31
        obs_group (Optional[str], optional): 项目ID. Defaults to None.
Wei Shoulin's avatar
Wei Shoulin committed
32
        obs_id (Optional[str], optional): 观测ID. Defaults to None.
33
        detector (Optional[str], optional): 探测器. Defaults to None.
34
        obs_type (Optional[str], optional): 观测类型,如'01'. Defaults to None.
Wei Shoulin's avatar
Wei Shoulin committed
35
36
37
38
39
40
41
42
43
44
        filter (Optional[str], optional): 滤光片. Defaults to None.
        obs_time (Optional[DateTimeTuple], optional): 观测时间范围. Defaults to None.
        create_time (Optional[DateTimeTuple], optional): 创建时间范围. Defaults to None.
        qc0_status (Optional[int], optional): QC0状态. Defaults to None.
        prc_status (Optional[int], optional): 处理状态. Defaults to None.
        file_name (Optional[str], optional): 文件名. Defaults to None.
        ra_obj (Optional[int], optional): 目标赤经. Defaults to None.
        dec_obj (Optional[int], optional): 目标赤纬. Defaults to None.
        radius (Optional[float], optional): 搜索半径. Defaults to None.
        object_name (Optional[str], optional): 目标名称. Defaults to None.
Wei Shoulin's avatar
Wei Shoulin committed
45
        dataset (Optional[str], optional): 数据集名称. Defaults to constants.DEFAULT_DATASET.
Wei Shoulin's avatar
Wei Shoulin committed
46
47
48
49
50
51
52
53
54
        page (int, optional): 页码. Defaults to 1.
        limit (int, optional): 每页数量. Defaults to 0,不限制.
    
    Returns:
        Result: 搜索结果对象.
    
    """

    params = {
55
        'obs_group': obs_group,
Wei Shoulin's avatar
Wei Shoulin committed
56
        'obs_id': obs_id,
Wei Shoulin's avatar
Wei Shoulin committed
57
        'instrument': instrument,
58
        'detector': detector,
59
        'obs_type': obs_type,
Wei Shoulin's avatar
Wei Shoulin committed
60
61
62
63
64
65
66
67
68
69
70
71
        'filter': filter,
        'qc0_status': qc0_status,
        'prc_status': prc_status,
        'file_name': file_name,
        'ra_obj': ra_obj,
        'dec_obj': dec_obj,
        'radius': radius,
        'object_name': object_name,
        'obs_time_start': None,
        'obs_time_end': None,
        'create_time_start': None,
        'create_time_end': None,
Wei Shoulin's avatar
Wei Shoulin committed
72
        'dataset': dataset,
Wei Shoulin's avatar
Wei Shoulin committed
73
74
75
76
77
78
        'page': page,
        'limit': limit,
    }
    
    if obs_time is not None:
        params['obs_time_start'], params['obs_time_end'] = obs_time
Wei Shoulin's avatar
Wei Shoulin committed
79
80
81
82
        if params['obs_time_start'] and utils.is_valid_datetime_format(params['obs_time_start']):
            pass
        if params['obs_time_end'] and utils.is_valid_datetime_format(params['obs_time_end']):
            pass 
Wei Shoulin's avatar
Wei Shoulin committed
83
84
    if create_time is not None:
        params['create_time_start'], params['create_time_end'] = create_time
Wei Shoulin's avatar
Wei Shoulin committed
85
        utils.is_valid_datetime_format(params['create_time_start']) or utils.is_valid_datetime_format(params['create_time_end'])
Wei Shoulin's avatar
Wei Shoulin committed
86
87
88
    return request.post("/api/level0", params)

def get_by_id(_id: str) -> Result:
Wei Shoulin's avatar
docs    
Wei Shoulin committed
89
90
91
92
93
94
95
96
97
98
    """
    根据内部ID获取0级数据
    
    Args:
        _id (str): 0级数据的内部ID
    
    Returns:
        Result: 查询结果
    
    """
Wei Shoulin's avatar
Wei Shoulin committed
99
100
101
102
    return request.get(f"/api/level0/_id/{_id}")

def find_by_level0_id(level0_id: str) -> Result:
    """
Wei Shoulin's avatar
Wei Shoulin committed
103
    通过 level0 的 ID 查询0级数据
Wei Shoulin's avatar
Wei Shoulin committed
104
105
    
    Args:
Wei Shoulin's avatar
Wei Shoulin committed
106
        level0_id (str): 0级数据的ID
Wei Shoulin's avatar
Wei Shoulin committed
107
108
    
    Returns:
Wei Shoulin's avatar
Wei Shoulin committed
109
        Result: 查询结果
Wei Shoulin's avatar
Wei Shoulin committed
110
111
112
113
    
    """
    return request.get(f"/api/level0/{level0_id}")

Wei Shoulin's avatar
Wei Shoulin committed
114
def update_qc0_status(level0_id: str, qc0_status: int, dataset: str = constants.DEFAULT_DATASET) -> Result:
Wei Shoulin's avatar
Wei Shoulin committed
115
116
117
118
119
120
    """
    更新0级数据的QC0状态
    
    Args:
        level0_id (str): 0级数据的ID
        qc0_status (int): QC0状态
Wei Shoulin's avatar
Wei Shoulin committed
121
        dataset (str): 数据集名称
Wei Shoulin's avatar
Wei Shoulin committed
122
123
124
125
    
    Returns:
        Result: 更新结果
    """
Wei Shoulin's avatar
Wei Shoulin committed
126
    return request.put(f"/api/level0/qc0_status/{level0_id}", {'qc0_status': qc0_status, 'dataset': dataset})
Wei Shoulin's avatar
Wei Shoulin committed
127

128
def update_prc_status(level0_id: str, dag_run: str, prc_status: int, dataset: str = constants.DEFAULT_DATASET) -> Result:
Wei Shoulin's avatar
Wei Shoulin committed
129
    """
Wei Shoulin's avatar
Wei Shoulin committed
130
    更新0级数据的处理状态
Wei Shoulin's avatar
Wei Shoulin committed
131
132
    
    Args:
Wei Shoulin's avatar
Wei Shoulin committed
133
        level0_id (str): 0级数据的ID
134
        dag_run (str): DAG运行标识
Wei Shoulin's avatar
Wei Shoulin committed
135
136
        prc_status (int): 处理状态
        dataset (str): 数据集名称
Wei Shoulin's avatar
Wei Shoulin committed
137
138
    
    Returns:
Wei Shoulin's avatar
Wei Shoulin committed
139
        Result: 操作结果
Wei Shoulin's avatar
Wei Shoulin committed
140
    """
141
    return request.put(f"/api/level0/prc_status/{level0_id}/{dag_run}", {'prc_status': prc_status, 'dataset': dataset})
Wei Shoulin's avatar
Wei Shoulin committed
142

Wei Shoulin's avatar
Wei Shoulin committed
143
144
145
def write(local_file: str, 
        dataset: str = constants.DEFAULT_DATASET,
        **kwargs) -> Result:
Wei Shoulin's avatar
Wei Shoulin committed
146
    """
Wei Shoulin's avatar
Wei Shoulin committed
147
    将本地文件写入DFS中
Wei Shoulin's avatar
Wei Shoulin committed
148
149
    
    Args:
Wei Shoulin's avatar
Wei Shoulin committed
150
151
152
        local_file (str]): 文件路径
        dataset (Optional[str], optional): 数据集名称. Defaults to None.
        **kwargs: 额外的关键字参数,这些参数将传递给DFS
Wei Shoulin's avatar
Wei Shoulin committed
153
154
    
    Returns:
Wei Shoulin's avatar
Wei Shoulin committed
155
        Result: 操作的结果对象,包含操作是否成功以及相关的错误信息,成功返回数据对象
Wei Shoulin's avatar
Wei Shoulin committed
156
157
    
    """
Wei Shoulin's avatar
Wei Shoulin committed
158
159
160
161
    params = {
        'dataset': dataset,
    }
    params.update(kwargs)
Wei Shoulin's avatar
Wei Shoulin committed
162
163
    if not os.path.exists(local_file):
        raise FileNotFoundError(local_file)        
Wei Shoulin's avatar
Wei Shoulin committed
164
    return request.post_file("/api/level0/file", local_file, params)
Wei Shoulin's avatar
Wei Shoulin committed
165

Wei Shoulin's avatar
Wei Shoulin committed
166
167
168
def write_cat(local_file: str,
            dataset: str = constants.DEFAULT_DATASET,
            **kwargs) -> Result:
Wei Shoulin's avatar
Wei Shoulin committed
169
    """
Wei Shoulin's avatar
Wei Shoulin committed
170
    主巡天仿真数据的星表本地文件写入DFS中
Wei Shoulin's avatar
Wei Shoulin committed
171
172
    
    Args:
Wei Shoulin's avatar
Wei Shoulin committed
173
174
175
        local_file (str]): 文件路径
        dataset (Optional[str], optional): 数据集名称. Defaults to None.
        **kwargs: 额外的关键字参数,这些参数将传递给DFS
Wei Shoulin's avatar
Wei Shoulin committed
176
177
    
    Returns:
Wei Shoulin's avatar
Wei Shoulin committed
178
        Result: 操作的结果对象,包含操作是否成功以及相关的错误信息,成功返回数据对象
Wei Shoulin's avatar
Wei Shoulin committed
179
180
    
    """
Wei Shoulin's avatar
Wei Shoulin committed
181
182
183
184
    params = {
        'dataset': dataset,
    }
    params.update(kwargs)
Wei Shoulin's avatar
Wei Shoulin committed
185
186
    if not os.path.exists(local_file):
        raise FileNotFoundError(local_file)        
Wei Shoulin's avatar
Wei Shoulin committed
187
    return request.post_file("/api/level0/cat/file", local_file, params)
Wei Shoulin's avatar
Wei Shoulin committed
188

189
190
def find_process(dag: Optional[str] = None, 
                dag_run: Optional[str] = None, 
Wei Shoulin's avatar
Wei Shoulin committed
191
192
193
194
195
196
197
198
                batch_id: Optional[str] = None, 
                level0_id: Optional[str] = None,
                dataset: Optional[str] = None,
                prc_module: Optional[str] = None,
                prc_status: Optional[int] = None,
                prc_time: Optional[DateTimeTuple] = None,
                page: int = 1,
                limit: int = 0) -> Result:
Wei Shoulin's avatar
Wei Shoulin committed
199
    """
Wei Shoulin's avatar
Wei Shoulin committed
200
    查询0级数据处理过程
Wei Shoulin's avatar
Wei Shoulin committed
201
202
    
    Args:
203
204
        dag (str): DAG标识
        dag_run (str): DAG运行标识
Wei Shoulin's avatar
Wei Shoulin committed
205
        batch_id (str): 批次ID
Wei Shoulin's avatar
Wei Shoulin committed
206
        level0_id (str): 0级数据的ID
Wei Shoulin's avatar
Wei Shoulin committed
207
208
209
210
211
212
        dataset (str): 数据集
        prc_module (str): 处理模块
        prc_status (int): 处理状态
        prc_time (DateTimeTuple): 处理时间范围
        page (int): 页码,默认为1
        limit (int): 每页数量 0: 不限制
Wei Shoulin's avatar
Wei Shoulin committed
213
214
    
    Returns:
Wei Shoulin's avatar
Wei Shoulin committed
215
        Result: 成功后,Result.data为数据列表,失败message为失败原因
Wei Shoulin's avatar
Wei Shoulin committed
216
217
    
    """
Wei Shoulin's avatar
Wei Shoulin committed
218
    params = {
219
220
        'dag': dag,
        'dag_run': dag_run,
Wei Shoulin's avatar
Wei Shoulin committed
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
        'batch_id': batch_id,
        'level0_id': level0_id,
        'dataset': dataset,
        'prc_module': prc_module,
        'prc_status': prc_status,
        'prc_time_start': None,
        'prc_time_end': None,
        'page': page,
        'limit': limit
    }
    if prc_time is not None:
        params['prc_time_start'], params['prc_time_end'] = prc_time
        if params['prc_time_start'] and utils.is_valid_datetime_format(params['prc_time_start']):
            pass
        if params['prc_time_end'] and utils.is_valid_datetime_format(params['prc_time_end']):
            pass 
    return request.post("/api/level0/process", params)
Wei Shoulin's avatar
Wei Shoulin committed
238
239

def add_process(level0_id: str, 
240
241
                dag: str, 
                dag_run: str, 
Wei Shoulin's avatar
Wei Shoulin committed
242
243
                batch_id: Optional[str] = None,
                dataset: str = constants.DEFAULT_DATASET,
Wei Shoulin's avatar
Wei Shoulin committed
244
                prc_status: int = -1024,
Wei Shoulin's avatar
Wei Shoulin committed
245
                prc_time: str = utils.get_current_time(),                
Wei Shoulin's avatar
Wei Shoulin committed
246
247
248
                prc_module: str = "",
                message: str = "") -> Result:
    """
Wei Shoulin's avatar
Wei Shoulin committed
249
    添加0级数据处理过程
Wei Shoulin's avatar
Wei Shoulin committed
250
251
    
    Args:
Wei Shoulin's avatar
Wei Shoulin committed
252
        level0_id (str): 0级数据的ID
253
254
        dag (str): DAG标识
        dag_run (str): DAG运行标识
Wei Shoulin's avatar
Wei Shoulin committed
255
256
257
258
259
260
        dataset (str): 数据集
        batch_id (str): 批次ID
        prc_time (str): 处理时间,格式为"YYYY-MM-DD HH:MM:SS"
        prc_status (int): 处理状态
        prc_module (str): 处理模块
        message (str): 处理消息
Wei Shoulin's avatar
Wei Shoulin committed
261
262
    
    Returns:
Wei Shoulin's avatar
Wei Shoulin committed
263
        Result: 成功后,Result.data为写入记录,失败message为失败原因
Wei Shoulin's avatar
Wei Shoulin committed
264
265
266
267
    
    """
    params = {
        'level0_id': level0_id,
268
269
        'dag': dag,
        'dag_run': dag_run,
Wei Shoulin's avatar
Wei Shoulin committed
270
271
        'dataset': dataset,
        'batch_id': batch_id,
Wei Shoulin's avatar
Wei Shoulin committed
272
273
274
275
276
        'prc_time': prc_time,
        'prc_status': prc_status,
        'prc_module': prc_module,
        'message': message,
    }
Wei Shoulin's avatar
Wei Shoulin committed
277
    utils.is_valid_datetime_format(prc_time)
Wei Shoulin's avatar
Wei Shoulin committed
278
279
280
281
    return request.post("/api/level0/prc", params)

def new(data: dict) -> Result:
    """
Wei Shoulin's avatar
Wei Shoulin committed
282
    新建0级数据,用于仿真数据测试
Wei Shoulin's avatar
Wei Shoulin committed
283
284
    
    Args:
Wei Shoulin's avatar
Wei Shoulin committed
285
        data (dict): 0级数据的字典表示
Wei Shoulin's avatar
Wei Shoulin committed
286
287
    
    Returns:
Wei Shoulin's avatar
Wei Shoulin committed
288
        Result: 成功后,Result.data为写入记录,失败message为失败原因
Wei Shoulin's avatar
Wei Shoulin committed
289
290
291
    
    """
    return request.post("/api/level0/new", data)