level0.py 11.2 KB
Newer Older
Wei Shoulin's avatar
Wei Shoulin committed
1
import os
2
3
4
from typing import List, Literal, Optional, Tuple

from .common import Result, constants, request, utils
Wei Shoulin's avatar
Wei Shoulin committed
5
6
7

DateTimeTuple = Tuple[str, str]

8

9
def find(
10
11
12
13
14
15
16
17
18
19
20
21
22
23
    dataset: str,
    instrument: Literal["MSC", "IFS", "MCI", "HSTDM", "CPIC"],
    data_model: str = "raw",
    data_uuid: Optional[str] = None,
    obs_group: Optional[str] = None,
    obs_id: Optional[str] = None,
    detector: Optional[str] = None,
    obs_type: Optional[str] = None,
    filter: Optional[str] = None,
    obs_date: Optional[DateTimeTuple] = None,
    create_time: Optional[DateTimeTuple] = None,
    qc_status: Optional[int] = None,
    prc_status: Optional[int] = None,
    file_name: Optional[str] = None,
24
25
    ra: Optional[int] = None,
    dec: Optional[int] = None,
26
27
28
29
30
31
    radius: Optional[float] = None,
    object: Optional[str] = None,
    page: int = 1,
    limit: int = 0,
    **extra_kwargs,
) -> Result:
Wei Shoulin's avatar
Wei Shoulin committed
32
    """
Wei Shoulin's avatar
Wei Shoulin committed
33
    根据给定的参数搜索0级数据文件记录
34

Wei Shoulin's avatar
Wei Shoulin committed
35
    Args:
36
37
        data_uuid (Optional[str], optional): 数据UUID. Defaults to None.
        data_model (str): 数据模型. Defaults is 'raw'.
38
        instrument (str): 设备,必需为'MSC', 'IFS', 'MCI', 'HSTDM', 'CPIC'之一.
39
        obs_group (Optional[str], optional): 项目ID. Defaults to None.
Wei Shoulin's avatar
Wei Shoulin committed
40
        obs_id (Optional[str], optional): 观测ID. Defaults to None.
41
        detector (Optional[str], optional): 探测器. Defaults to None.
42
        obs_type (Optional[str], optional): 观测类型,如'01'. Defaults to None.
Wei Shoulin's avatar
Wei Shoulin committed
43
        filter (Optional[str], optional): 滤光片. Defaults to None.
44
        obs_date (Optional[DateTimeTuple], optional): 观测时间范围. Defaults to None.
Wei Shoulin's avatar
Wei Shoulin committed
45
        create_time (Optional[DateTimeTuple], optional): 创建时间范围. Defaults to None.
46
        qc_status (Optional[int], optional): QC0状态. Defaults to None.
Wei Shoulin's avatar
Wei Shoulin committed
47
48
        prc_status (Optional[int], optional): 处理状态. Defaults to None.
        file_name (Optional[str], optional): 文件名. Defaults to None.
49
50
        ra (Optional[int], optional): 目标赤经. Defaults to None.
        dec (Optional[int], optional): 目标赤纬. Defaults to None.
Wei Shoulin's avatar
Wei Shoulin committed
51
        radius (Optional[float], optional): 搜索半径. Defaults to None.
52
        object (Optional[str], optional): 目标名称. Defaults to None.
Wei Shoulin's avatar
Wei Shoulin committed
53
        dataset (Optional[str], optional): 数据集名称. Defaults to constants.DEFAULT_DATASET.
Wei Shoulin's avatar
Wei Shoulin committed
54
55
        page (int, optional): 页码. Defaults to 1.
        limit (int, optional): 每页数量. Defaults to 0,不限制.
56

Wei Shoulin's avatar
Wei Shoulin committed
57
58
    Returns:
        Result: 搜索结果对象.
59

Wei Shoulin's avatar
Wei Shoulin committed
60
61
62
    """

    params = {
63
64
65
66
67
68
69
70
71
72
73
        "data_uuid": data_uuid,
        "data_model": data_model,
        "obs_group": obs_group,
        "obs_id": obs_id,
        "instrument": instrument,
        "detector": detector,
        "obs_type": obs_type,
        "filter": filter,
        "qc_status": qc_status,
        "prc_status": prc_status,
        "file_name": file_name,
74
75
        "ra_obj": ra,
        "dec_obj": dec,
76
77
78
79
80
81
82
83
84
        "radius": radius,
        "object": object,
        "obs_date_start": None,
        "obs_date_end": None,
        "create_time_start": None,
        "create_time_end": None,
        "dataset": dataset,
        "page": page,
        "limit": limit,
Wei Shoulin's avatar
Wei Shoulin committed
85
    }
86
87
    params.update(extra_kwargs)

88
    if obs_date is not None:
89
90
91
92
93
94
95
96
        params["obs_date_start"], params["obs_date_end"] = obs_date
        if params["obs_date_start"] and utils.is_valid_datetime_format(
            params["obs_date_start"]
        ):
            pass
        if params["obs_date_end"] and utils.is_valid_datetime_format(
            params["obs_date_end"]
        ):
Wei Shoulin's avatar
Wei Shoulin committed
97
            pass
Wei Shoulin's avatar
Wei Shoulin committed
98
    if create_time is not None:
99
100
101
102
        params["create_time_start"], params["create_time_end"] = create_time
        utils.is_valid_datetime_format(
            params["create_time_start"]
        ) or utils.is_valid_datetime_format(params["create_time_end"])
Wei Shoulin's avatar
Wei Shoulin committed
103
104
    return request.post("/api/level0", params)

105

Wei Shoulin's avatar
Wei Shoulin committed
106
def get_by_id(_id: str) -> Result:
Wei Shoulin's avatar
docs    
Wei Shoulin committed
107
108
    """
    根据内部ID获取0级数据
109

Wei Shoulin's avatar
docs    
Wei Shoulin committed
110
111
    Args:
        _id (str): 0级数据的内部ID
112

Wei Shoulin's avatar
docs    
Wei Shoulin committed
113
114
    Returns:
        Result: 查询结果
115

Wei Shoulin's avatar
docs    
Wei Shoulin committed
116
    """
Wei Shoulin's avatar
Wei Shoulin committed
117
118
    return request.get(f"/api/level0/_id/{_id}")

119
120
121
122
123
124
125
126
127
128
129
130
131
132
133

def get_by_uuid(_uuid: str) -> Result:
    """
    根据UUID获取0级数据

    Args:
        _uuid (str): 0级数据的UUID

    Returns:
        Result: 查询结果

    """
    return request.get(f"/api/level0/uuid/{_uuid}")


Wei Shoulin's avatar
Wei Shoulin committed
134
135
def find_by_level0_id(level0_id: str) -> Result:
    """
Wei Shoulin's avatar
Wei Shoulin committed
136
    通过 level0 的 ID 查询0级数据
137

Wei Shoulin's avatar
Wei Shoulin committed
138
    Args:
Wei Shoulin's avatar
Wei Shoulin committed
139
        level0_id (str): 0级数据的ID
140

Wei Shoulin's avatar
Wei Shoulin committed
141
    Returns:
Wei Shoulin's avatar
Wei Shoulin committed
142
        Result: 查询结果
143

Wei Shoulin's avatar
Wei Shoulin committed
144
145
146
    """
    return request.get(f"/api/level0/{level0_id}")

147

148
def update_qc_status(level0_id: str, qc_status: int, dataset: str) -> Result:
Wei Shoulin's avatar
Wei Shoulin committed
149
    """
150
    更新0级数据的QC状态
151

Wei Shoulin's avatar
Wei Shoulin committed
152
153
    Args:
        level0_id (str): 0级数据的ID
154
        qc_status (int): QC状态
Wei Shoulin's avatar
Wei Shoulin committed
155
        dataset (str): 数据集名称
156

Wei Shoulin's avatar
Wei Shoulin committed
157
158
159
    Returns:
        Result: 更新结果
    """
160
161
162
163
164
    return request.put(
        f"/api/level0/qc_status/{level0_id}",
        {"qc_status": qc_status, "dataset": dataset},
    )

Wei Shoulin's avatar
Wei Shoulin committed
165

166
def update_qc_status_by_ids(ids: List[str], qc_status: int) -> Result:
167
    """
168
    根据内部_id,批量更新0级数据的QC状态
169

170
171
    Args:
        ids (List[str]): 内部_id列表
172
        qc_status (int): QC状态
173

174
175
176
    Returns:
        Result: 更新结果
    """
177
178
179
    return request.put(
        "/api/level0/qc_status/batch/update", {"qc_status": qc_status, "ids": ids}
    )
180

181
182
183
184

def update_prc_status(
    level0_id: str, dag_run: str, prc_status: int, dataset: str
) -> Result:
Wei Shoulin's avatar
Wei Shoulin committed
185
    """
Wei Shoulin's avatar
Wei Shoulin committed
186
    更新0级数据的处理状态
187

Wei Shoulin's avatar
Wei Shoulin committed
188
    Args:
Wei Shoulin's avatar
Wei Shoulin committed
189
        level0_id (str): 0级数据的ID
190
        dag_run (str): DAG运行标识
Wei Shoulin's avatar
Wei Shoulin committed
191
192
        prc_status (int): 处理状态
        dataset (str): 数据集名称
193

Wei Shoulin's avatar
Wei Shoulin committed
194
    Returns:
Wei Shoulin's avatar
Wei Shoulin committed
195
        Result: 操作结果
Wei Shoulin's avatar
Wei Shoulin committed
196
    """
197
198
199
200
201
    return request.put(
        f"/api/level0/prc_status/{level0_id}/{dag_run}",
        {"prc_status": prc_status, "dataset": dataset},
    )

Wei Shoulin's avatar
Wei Shoulin committed
202

203
204
205
def update_prc_status_by_ids(ids: List[str], prc_status: int) -> Result:
    """
    根据内部_id,批量更新0级数据的处理状态
206

207
208
209
    Args:
        ids (List[str]): 内部_id列表
        prc_status (int): 处理状态
210

211
212
213
    Returns:
        Result: 操作结果
    """
214
215
216
    return request.put(
        "/api/level0/prc_status/batch", {"prc_status": prc_status, "ids": ids}
    )
217

218
219
220
221

def write(
    local_file: str, dataset: str = constants.DEFAULT_DATASET, **kwargs
) -> Result:
Wei Shoulin's avatar
Wei Shoulin committed
222
    """
Wei Shoulin's avatar
Wei Shoulin committed
223
    将本地文件写入DFS中
224

Wei Shoulin's avatar
Wei Shoulin committed
225
    Args:
Wei Shoulin's avatar
Wei Shoulin committed
226
227
228
        local_file (str]): 文件路径
        dataset (Optional[str], optional): 数据集名称. Defaults to None.
        **kwargs: 额外的关键字参数,这些参数将传递给DFS
229

Wei Shoulin's avatar
Wei Shoulin committed
230
    Returns:
Wei Shoulin's avatar
Wei Shoulin committed
231
        Result: 操作的结果对象,包含操作是否成功以及相关的错误信息,成功返回数据对象
232

Wei Shoulin's avatar
Wei Shoulin committed
233
    """
Wei Shoulin's avatar
Wei Shoulin committed
234
    params = {
235
        "dataset": dataset,
Wei Shoulin's avatar
Wei Shoulin committed
236
237
    }
    params.update(kwargs)
Wei Shoulin's avatar
Wei Shoulin committed
238
    if not os.path.exists(local_file):
239
        raise FileNotFoundError(local_file)
Wei Shoulin's avatar
Wei Shoulin committed
240
    return request.post_file("/api/level0/file", local_file, params)
Wei Shoulin's avatar
Wei Shoulin committed
241

242
243
244
245

def write_cat(
    local_file: str, dataset: str = constants.DEFAULT_DATASET, **kwargs
) -> Result:
Wei Shoulin's avatar
Wei Shoulin committed
246
    """
Wei Shoulin's avatar
Wei Shoulin committed
247
    主巡天仿真数据的星表本地文件写入DFS中
248

Wei Shoulin's avatar
Wei Shoulin committed
249
    Args:
Wei Shoulin's avatar
Wei Shoulin committed
250
251
252
        local_file (str]): 文件路径
        dataset (Optional[str], optional): 数据集名称. Defaults to None.
        **kwargs: 额外的关键字参数,这些参数将传递给DFS
253

Wei Shoulin's avatar
Wei Shoulin committed
254
    Returns:
Wei Shoulin's avatar
Wei Shoulin committed
255
        Result: 操作的结果对象,包含操作是否成功以及相关的错误信息,成功返回数据对象
256

Wei Shoulin's avatar
Wei Shoulin committed
257
    """
Wei Shoulin's avatar
Wei Shoulin committed
258
    params = {
259
        "dataset": dataset,
Wei Shoulin's avatar
Wei Shoulin committed
260
261
    }
    params.update(kwargs)
Wei Shoulin's avatar
Wei Shoulin committed
262
    if not os.path.exists(local_file):
263
        raise FileNotFoundError(local_file)
Wei Shoulin's avatar
Wei Shoulin committed
264
    return request.post_file("/api/level0/cat/file", local_file, params)
Wei Shoulin's avatar
Wei Shoulin committed
265

266
267
268
269
270
271
272
273
274
275
276
277
278

def find_process(
    dag: Optional[str] = None,
    dag_run: Optional[str] = None,
    batch_id: Optional[str] = None,
    level0_id: Optional[str] = None,
    dataset: Optional[str] = None,
    prc_module: Optional[str] = None,
    prc_status: Optional[int] = None,
    prc_date: Optional[DateTimeTuple] = None,
    page: int = 1,
    limit: int = 0,
) -> Result:
Wei Shoulin's avatar
Wei Shoulin committed
279
    """
Wei Shoulin's avatar
Wei Shoulin committed
280
    查询0级数据处理过程
281

Wei Shoulin's avatar
Wei Shoulin committed
282
    Args:
283
284
        dag (str): DAG标识
        dag_run (str): DAG运行标识
Wei Shoulin's avatar
Wei Shoulin committed
285
        batch_id (str): 批次ID
Wei Shoulin's avatar
Wei Shoulin committed
286
        level0_id (str): 0级数据的ID
Wei Shoulin's avatar
Wei Shoulin committed
287
288
289
        dataset (str): 数据集
        prc_module (str): 处理模块
        prc_status (int): 处理状态
290
        prc_date (DateTimeTuple): 处理时间范围
Wei Shoulin's avatar
Wei Shoulin committed
291
292
        page (int): 页码,默认为1
        limit (int): 每页数量 0: 不限制
293

Wei Shoulin's avatar
Wei Shoulin committed
294
    Returns:
Wei Shoulin's avatar
Wei Shoulin committed
295
        Result: 成功后,Result.data为数据列表,失败message为失败原因
296

Wei Shoulin's avatar
Wei Shoulin committed
297
    """
Wei Shoulin's avatar
Wei Shoulin committed
298
    params = {
299
300
301
302
303
304
305
306
307
308
309
        "dag": dag,
        "dag_run": dag_run,
        "batch_id": batch_id,
        "level0_id": level0_id,
        "dataset": dataset,
        "prc_module": prc_module,
        "prc_status": prc_status,
        "prc_date_start": None,
        "prc_date_end": None,
        "page": page,
        "limit": limit,
Wei Shoulin's avatar
Wei Shoulin committed
310
    }
311
    if prc_date is not None:
312
313
314
315
316
317
318
319
        params["prc_date_start"], params["prc_date_end"] = prc_date
        if params["prc_date_start"] and utils.is_valid_datetime_format(
            params["prc_date_start"]
        ):
            pass
        if params["prc_date_end"] and utils.is_valid_datetime_format(
            params["prc_date_end"]
        ):
Wei Shoulin's avatar
Wei Shoulin committed
320
321
            pass
    return request.post("/api/level0/process", params)
Wei Shoulin's avatar
Wei Shoulin committed
322

323
324
325
326
327
328
329
330
331
332
333
334

def add_process(
    level0_id: str,
    dag: str,
    dag_run: str,
    batch_id: Optional[str] = None,
    dataset: str = constants.DEFAULT_DATASET,
    prc_status: int = -1024,
    prc_date: str = utils.get_current_time(),
    prc_module: str = "",
    message: str = "",
) -> Result:
Wei Shoulin's avatar
Wei Shoulin committed
335
    """
Wei Shoulin's avatar
Wei Shoulin committed
336
    添加0级数据处理过程
337

Wei Shoulin's avatar
Wei Shoulin committed
338
    Args:
Wei Shoulin's avatar
Wei Shoulin committed
339
        level0_id (str): 0级数据的ID
340
341
        dag (str): DAG标识
        dag_run (str): DAG运行标识
Wei Shoulin's avatar
Wei Shoulin committed
342
343
        dataset (str): 数据集
        batch_id (str): 批次ID
344
        prc_date (str): 处理时间,格式为"YYYY-MM-DD HH:MM:SS"
Wei Shoulin's avatar
Wei Shoulin committed
345
346
347
        prc_status (int): 处理状态
        prc_module (str): 处理模块
        message (str): 处理消息
348

Wei Shoulin's avatar
Wei Shoulin committed
349
    Returns:
Wei Shoulin's avatar
Wei Shoulin committed
350
        Result: 成功后,Result.data为写入记录,失败message为失败原因
351

Wei Shoulin's avatar
Wei Shoulin committed
352
353
    """
    params = {
354
355
356
357
358
359
360
361
362
        "level0_id": level0_id,
        "dag": dag,
        "dag_run": dag_run,
        "dataset": dataset,
        "batch_id": batch_id,
        "prc_date": prc_date,
        "prc_status": prc_status,
        "prc_module": prc_module,
        "message": message,
Wei Shoulin's avatar
Wei Shoulin committed
363
    }
364
    utils.is_valid_datetime_format(prc_date)
Wei Shoulin's avatar
Wei Shoulin committed
365
366
    return request.post("/api/level0/prc", params)

367

Wei Shoulin's avatar
Wei Shoulin committed
368
369
def new(data: dict) -> Result:
    """
Wei Shoulin's avatar
Wei Shoulin committed
370
    新建0级数据,用于仿真数据测试
371

Wei Shoulin's avatar
Wei Shoulin committed
372
    Args:
Wei Shoulin's avatar
Wei Shoulin committed
373
        data (dict): 0级数据的字典表示
374

Wei Shoulin's avatar
Wei Shoulin committed
375
    Returns:
Wei Shoulin's avatar
Wei Shoulin committed
376
        Result: 成功后,Result.data为写入记录,失败message为失败原因
377

Wei Shoulin's avatar
Wei Shoulin committed
378
    """
379
    return request.post("/api/level0/new", data)
380
381
382
383
384
385
386
387
388
389
390
391
392
393

def delete(dataset: str, data_model: str = 'raw') -> Result:
    """
    删除0级数据,用于仿真数据测试

    Args:
        dataset (str): 数据集
        data_model (str): 数据模型

    Returns:
        Result: 操作的结果对象,包含操作是否成功以及相关的错误信息,成功返回数据对象

    """
    return request.delete("/api/level0/delete", params = {"dataset": dataset, "data_model": data_model})