level0.py 9.86 KB
Newer Older
Wei Shoulin's avatar
Wei Shoulin committed
1
import os
Wei Shoulin's avatar
t    
Wei Shoulin committed
2
from typing import Optional, Tuple, Literal
Wei Shoulin's avatar
Wei Shoulin committed
3
from .common import request, Result, utils, constants
Wei Shoulin's avatar
Wei Shoulin committed
4
5
6

DateTimeTuple = Tuple[str, str]

7
def find(obs_group: Optional[str] = None,
Wei Shoulin's avatar
Wei Shoulin committed
8
        obs_id: Optional[str] = None,
Wei Shoulin's avatar
Wei Shoulin committed
9
        instrument: Literal['MSC', 'IFS', 'MCI', 'HSTDM', 'CPIC'] = 'MSC', 
10
        detector: Optional[str] = None,
11
        obs_type: Optional[str] = None,
Wei Shoulin's avatar
Wei Shoulin committed
12
13
14
15
16
17
18
19
20
21
        filter: Optional[str] = None,
        obs_time: Optional[DateTimeTuple] = None,
        create_time: Optional[DateTimeTuple] = None,
        qc0_status: Optional[int] = None,
        prc_status: Optional[int] = None,
        file_name: Optional[str] = None,
        ra_obj: Optional[int] = None,
        dec_obj: Optional[int] = None,
        radius: Optional[float] = None,
        object_name: Optional[str] = None,
Wei Shoulin's avatar
Wei Shoulin committed
22
        dataset: str = constants.DEFAULT_DATASET,
Wei Shoulin's avatar
Wei Shoulin committed
23
24
25
        page: int = 1,
        limit: int = 0) -> Result:
    """
Wei Shoulin's avatar
Wei Shoulin committed
26
    根据给定的参数搜索0级数据文件记录
Wei Shoulin's avatar
Wei Shoulin committed
27
28
    
    Args:
29
        obs_group (Optional[str], optional): 项目ID. Defaults to None.
Wei Shoulin's avatar
Wei Shoulin committed
30
        obs_id (Optional[str], optional): 观测ID. Defaults to None.
Wei Shoulin's avatar
Wei Shoulin committed
31
        instrument (Optional[str], optional): 设备,如'MSC', 'IFS'. Defaults to 'MSC'.
32
        detector (Optional[str], optional): 探测器. Defaults to None.
33
        obs_type (Optional[str], optional): 观测类型,如'01'. Defaults to None.
Wei Shoulin's avatar
Wei Shoulin committed
34
35
36
37
38
39
40
41
42
43
        filter (Optional[str], optional): 滤光片. Defaults to None.
        obs_time (Optional[DateTimeTuple], optional): 观测时间范围. Defaults to None.
        create_time (Optional[DateTimeTuple], optional): 创建时间范围. Defaults to None.
        qc0_status (Optional[int], optional): QC0状态. Defaults to None.
        prc_status (Optional[int], optional): 处理状态. Defaults to None.
        file_name (Optional[str], optional): 文件名. Defaults to None.
        ra_obj (Optional[int], optional): 目标赤经. Defaults to None.
        dec_obj (Optional[int], optional): 目标赤纬. Defaults to None.
        radius (Optional[float], optional): 搜索半径. Defaults to None.
        object_name (Optional[str], optional): 目标名称. Defaults to None.
Wei Shoulin's avatar
Wei Shoulin committed
44
        dataset (Optional[str], optional): 数据集名称. Defaults to constants.DEFAULT_DATASET.
Wei Shoulin's avatar
Wei Shoulin committed
45
46
47
48
49
50
51
52
53
        page (int, optional): 页码. Defaults to 1.
        limit (int, optional): 每页数量. Defaults to 0,不限制.
    
    Returns:
        Result: 搜索结果对象.
    
    """

    params = {
54
        'obs_group': obs_group,
Wei Shoulin's avatar
Wei Shoulin committed
55
        'obs_id': obs_id,
Wei Shoulin's avatar
Wei Shoulin committed
56
        'instrument': instrument,
57
        'detector': detector,
58
        'obs_type': obs_type,
Wei Shoulin's avatar
Wei Shoulin committed
59
60
61
62
63
64
65
66
67
68
69
70
        'filter': filter,
        'qc0_status': qc0_status,
        'prc_status': prc_status,
        'file_name': file_name,
        'ra_obj': ra_obj,
        'dec_obj': dec_obj,
        'radius': radius,
        'object_name': object_name,
        'obs_time_start': None,
        'obs_time_end': None,
        'create_time_start': None,
        'create_time_end': None,
Wei Shoulin's avatar
Wei Shoulin committed
71
        'dataset': dataset,
Wei Shoulin's avatar
Wei Shoulin committed
72
73
74
75
76
77
        'page': page,
        'limit': limit,
    }
    
    if obs_time is not None:
        params['obs_time_start'], params['obs_time_end'] = obs_time
Wei Shoulin's avatar
Wei Shoulin committed
78
79
80
81
        if params['obs_time_start'] and utils.is_valid_datetime_format(params['obs_time_start']):
            pass
        if params['obs_time_end'] and utils.is_valid_datetime_format(params['obs_time_end']):
            pass 
Wei Shoulin's avatar
Wei Shoulin committed
82
83
    if create_time is not None:
        params['create_time_start'], params['create_time_end'] = create_time
Wei Shoulin's avatar
Wei Shoulin committed
84
        utils.is_valid_datetime_format(params['create_time_start']) or utils.is_valid_datetime_format(params['create_time_end'])
Wei Shoulin's avatar
Wei Shoulin committed
85
86
87
    return request.post("/api/level0", params)

def get_by_id(_id: str) -> Result:
Wei Shoulin's avatar
docs    
Wei Shoulin committed
88
89
90
91
92
93
94
95
96
97
    """
    根据内部ID获取0级数据
    
    Args:
        _id (str): 0级数据的内部ID
    
    Returns:
        Result: 查询结果
    
    """
Wei Shoulin's avatar
Wei Shoulin committed
98
99
100
101
    return request.get(f"/api/level0/_id/{_id}")

def find_by_level0_id(level0_id: str) -> Result:
    """
Wei Shoulin's avatar
Wei Shoulin committed
102
    通过 level0 的 ID 查询0级数据
Wei Shoulin's avatar
Wei Shoulin committed
103
104
    
    Args:
Wei Shoulin's avatar
Wei Shoulin committed
105
        level0_id (str): 0级数据的ID
Wei Shoulin's avatar
Wei Shoulin committed
106
107
    
    Returns:
Wei Shoulin's avatar
Wei Shoulin committed
108
        Result: 查询结果
Wei Shoulin's avatar
Wei Shoulin committed
109
110
111
112
    
    """
    return request.get(f"/api/level0/{level0_id}")

Wei Shoulin's avatar
Wei Shoulin committed
113
def update_qc0_status(level0_id: str, qc0_status: int, dataset: str = constants.DEFAULT_DATASET) -> Result:
Wei Shoulin's avatar
Wei Shoulin committed
114
115
116
117
118
119
    """
    更新0级数据的QC0状态
    
    Args:
        level0_id (str): 0级数据的ID
        qc0_status (int): QC0状态
Wei Shoulin's avatar
Wei Shoulin committed
120
        dataset (str): 数据集名称
Wei Shoulin's avatar
Wei Shoulin committed
121
122
123
124
    
    Returns:
        Result: 更新结果
    """
Wei Shoulin's avatar
Wei Shoulin committed
125
    return request.put(f"/api/level0/qc0_status/{level0_id}", {'qc0_status': qc0_status, 'dataset': dataset})
Wei Shoulin's avatar
Wei Shoulin committed
126

Wei Shoulin's avatar
Wei Shoulin committed
127
def update_prc_status(level0_id: str, dag_run_id: str, prc_status: int, dataset: str = constants.DEFAULT_DATASET) -> Result:
Wei Shoulin's avatar
Wei Shoulin committed
128
    """
Wei Shoulin's avatar
Wei Shoulin committed
129
    更新0级数据的处理状态
Wei Shoulin's avatar
Wei Shoulin committed
130
131
    
    Args:
Wei Shoulin's avatar
Wei Shoulin committed
132
        level0_id (str): 0级数据的ID
133
        dag_run_id (str): 运行ID
Wei Shoulin's avatar
Wei Shoulin committed
134
135
        prc_status (int): 处理状态
        dataset (str): 数据集名称
Wei Shoulin's avatar
Wei Shoulin committed
136
137
    
    Returns:
Wei Shoulin's avatar
Wei Shoulin committed
138
        Result: 操作结果
Wei Shoulin's avatar
Wei Shoulin committed
139
    """
Wei Shoulin's avatar
Wei Shoulin committed
140
    return request.put(f"/api/level0/prc_status/{level0_id}/{dag_run_id}", {'prc_status': prc_status, 'dataset': dataset})
Wei Shoulin's avatar
Wei Shoulin committed
141

Wei Shoulin's avatar
Wei Shoulin committed
142
143
144
def write(local_file: str, 
        dataset: str = constants.DEFAULT_DATASET,
        **kwargs) -> Result:
Wei Shoulin's avatar
Wei Shoulin committed
145
    """
Wei Shoulin's avatar
Wei Shoulin committed
146
    将本地文件写入DFS中
Wei Shoulin's avatar
Wei Shoulin committed
147
148
    
    Args:
Wei Shoulin's avatar
Wei Shoulin committed
149
150
151
        local_file (str]): 文件路径
        dataset (Optional[str], optional): 数据集名称. Defaults to None.
        **kwargs: 额外的关键字参数,这些参数将传递给DFS
Wei Shoulin's avatar
Wei Shoulin committed
152
153
    
    Returns:
Wei Shoulin's avatar
Wei Shoulin committed
154
        Result: 操作的结果对象,包含操作是否成功以及相关的错误信息,成功返回数据对象
Wei Shoulin's avatar
Wei Shoulin committed
155
156
    
    """
Wei Shoulin's avatar
Wei Shoulin committed
157
158
159
160
    params = {
        'dataset': dataset,
    }
    params.update(kwargs)
Wei Shoulin's avatar
Wei Shoulin committed
161
162
    if not os.path.exists(local_file):
        raise FileNotFoundError(local_file)        
Wei Shoulin's avatar
Wei Shoulin committed
163
    return request.post_file("/api/level0/file", local_file, params)
Wei Shoulin's avatar
Wei Shoulin committed
164

Wei Shoulin's avatar
Wei Shoulin committed
165
166
167
def write_cat(local_file: str,
            dataset: str = constants.DEFAULT_DATASET,
            **kwargs) -> Result:
Wei Shoulin's avatar
Wei Shoulin committed
168
    """
Wei Shoulin's avatar
Wei Shoulin committed
169
    主巡天仿真数据的星表本地文件写入DFS中
Wei Shoulin's avatar
Wei Shoulin committed
170
171
    
    Args:
Wei Shoulin's avatar
Wei Shoulin committed
172
173
174
        local_file (str]): 文件路径
        dataset (Optional[str], optional): 数据集名称. Defaults to None.
        **kwargs: 额外的关键字参数,这些参数将传递给DFS
Wei Shoulin's avatar
Wei Shoulin committed
175
176
    
    Returns:
Wei Shoulin's avatar
Wei Shoulin committed
177
        Result: 操作的结果对象,包含操作是否成功以及相关的错误信息,成功返回数据对象
Wei Shoulin's avatar
Wei Shoulin committed
178
179
    
    """
Wei Shoulin's avatar
Wei Shoulin committed
180
181
182
183
    params = {
        'dataset': dataset,
    }
    params.update(kwargs)
Wei Shoulin's avatar
Wei Shoulin committed
184
185
    if not os.path.exists(local_file):
        raise FileNotFoundError(local_file)        
Wei Shoulin's avatar
Wei Shoulin committed
186
    return request.post_file("/api/level0/cat/file", local_file, params)
Wei Shoulin's avatar
Wei Shoulin committed
187

Wei Shoulin's avatar
Wei Shoulin committed
188
189
190
191
192
193
194
195
196
197
def find_process(dag_id: Optional[str] = None, 
                dag_run_id: Optional[str] = None, 
                batch_id: Optional[str] = None, 
                level0_id: Optional[str] = None,
                dataset: Optional[str] = None,
                prc_module: Optional[str] = None,
                prc_status: Optional[int] = None,
                prc_time: Optional[DateTimeTuple] = None,
                page: int = 1,
                limit: int = 0) -> Result:
Wei Shoulin's avatar
Wei Shoulin committed
198
    """
Wei Shoulin's avatar
Wei Shoulin committed
199
    查询0级数据处理过程
Wei Shoulin's avatar
Wei Shoulin committed
200
201
    
    Args:
Wei Shoulin's avatar
Wei Shoulin committed
202
203
204
        dag_id (str): 管线ID
        dag_run_id (str): 运行ID
        batch_id (str): 批次ID
Wei Shoulin's avatar
Wei Shoulin committed
205
        level0_id (str): 0级数据的ID
Wei Shoulin's avatar
Wei Shoulin committed
206
207
208
209
210
211
        dataset (str): 数据集
        prc_module (str): 处理模块
        prc_status (int): 处理状态
        prc_time (DateTimeTuple): 处理时间范围
        page (int): 页码,默认为1
        limit (int): 每页数量 0: 不限制
Wei Shoulin's avatar
Wei Shoulin committed
212
213
    
    Returns:
Wei Shoulin's avatar
Wei Shoulin committed
214
        Result: 成功后,Result.data为数据列表,失败message为失败原因
Wei Shoulin's avatar
Wei Shoulin committed
215
216
    
    """
Wei Shoulin's avatar
Wei Shoulin committed
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
    params = {
        'dag_id': dag_id,
        'dag_run_id': dag_run_id,
        'batch_id': batch_id,
        'level0_id': level0_id,
        'dataset': dataset,
        'prc_module': prc_module,
        'prc_status': prc_status,
        'prc_time_start': None,
        'prc_time_end': None,
        'page': page,
        'limit': limit
    }
    if prc_time is not None:
        params['prc_time_start'], params['prc_time_end'] = prc_time
        if params['prc_time_start'] and utils.is_valid_datetime_format(params['prc_time_start']):
            pass
        if params['prc_time_end'] and utils.is_valid_datetime_format(params['prc_time_end']):
            pass 
    return request.post("/api/level0/process", params)
Wei Shoulin's avatar
Wei Shoulin committed
237
238

def add_process(level0_id: str, 
239
240
                dag_id: str, 
                dag_run_id: str, 
Wei Shoulin's avatar
Wei Shoulin committed
241
242
                batch_id: Optional[str] = None,
                dataset: str = constants.DEFAULT_DATASET,
Wei Shoulin's avatar
Wei Shoulin committed
243
                prc_status: int = -1024,
Wei Shoulin's avatar
Wei Shoulin committed
244
                prc_time: str = utils.get_current_time(),                
Wei Shoulin's avatar
Wei Shoulin committed
245
246
247
                prc_module: str = "",
                message: str = "") -> Result:
    """
Wei Shoulin's avatar
Wei Shoulin committed
248
    添加0级数据处理过程
Wei Shoulin's avatar
Wei Shoulin committed
249
250
    
    Args:
Wei Shoulin's avatar
Wei Shoulin committed
251
        level0_id (str): 0级数据的ID
252
253
        dag_id (str): 管线ID
        dag_run_id (str): 运行ID
Wei Shoulin's avatar
Wei Shoulin committed
254
255
256
257
258
259
        dataset (str): 数据集
        batch_id (str): 批次ID
        prc_time (str): 处理时间,格式为"YYYY-MM-DD HH:MM:SS"
        prc_status (int): 处理状态
        prc_module (str): 处理模块
        message (str): 处理消息
Wei Shoulin's avatar
Wei Shoulin committed
260
261
    
    Returns:
Wei Shoulin's avatar
Wei Shoulin committed
262
        Result: 成功后,Result.data为写入记录,失败message为失败原因
Wei Shoulin's avatar
Wei Shoulin committed
263
264
265
266
    
    """
    params = {
        'level0_id': level0_id,
267
268
        'dag_id': dag_id,
        'dag_run_id': dag_run_id,
Wei Shoulin's avatar
Wei Shoulin committed
269
270
        'dataset': dataset,
        'batch_id': batch_id,
Wei Shoulin's avatar
Wei Shoulin committed
271
272
273
274
275
        'prc_time': prc_time,
        'prc_status': prc_status,
        'prc_module': prc_module,
        'message': message,
    }
Wei Shoulin's avatar
Wei Shoulin committed
276
    utils.is_valid_datetime_format(prc_time)
Wei Shoulin's avatar
Wei Shoulin committed
277
278
279
280
    return request.post("/api/level0/prc", params)

def new(data: dict) -> Result:
    """
Wei Shoulin's avatar
Wei Shoulin committed
281
    新建0级数据,用于仿真数据测试
Wei Shoulin's avatar
Wei Shoulin committed
282
283
    
    Args:
Wei Shoulin's avatar
Wei Shoulin committed
284
        data (dict): 0级数据的字典表示
Wei Shoulin's avatar
Wei Shoulin committed
285
286
    
    Returns:
Wei Shoulin's avatar
Wei Shoulin committed
287
        Result: 成功后,Result.data为写入记录,失败message为失败原因
Wei Shoulin's avatar
Wei Shoulin committed
288
289
290
    
    """
    return request.post("/api/level0/new", data)