Skip to content
GitLab
Commit 5590255b authored by BO ZHANG's avatar BO ZHANG 🏀
Browse files

delete unnecessary files

parent 55e4b081
Pipeline #1464 passed with stage
in 0 seconds
Hide whitespace changes
Inline Side-by-side
docs/source/ch10_integration.rst deleted 100644 → 0
View file @ 55e4b081
Continuous Integration
======================
Using ``Jenkins``
-----------------
to be updated
Base ``docker`` image
---------------------
The CSST pipeline will be dockerized from the base image ``continuumio/anaconda3``:
- https://hub.docker.com/r/continuumio/anaconda3
The basic information is shown below
.. code-block::
# cat /etc/issue
Debian GNU/Linux 11 \n \l
# which python
/opt/conda/bin/python
# which conda
/opt/conda/bin/conda
# which pip
/opt/conda/bin/pip
# conda info
active environment : None
user config file : /root/.condarc
populated config files :
conda version : 4.12.0
conda-build version : 3.21.8
python version : 3.9.12.final.0
virtual packages : __linux=5.10.124=0
__glibc=2.31=0
__unix=0=0
__archspec=1=aarch64
base environment : /opt/conda (writable)
conda av data dir : /opt/conda/etc/conda
conda av metadata url : None
channel URLs : https://repo.anaconda.com/pkgs/main/linux-aarch64
https://repo.anaconda.com/pkgs/main/noarch
https://repo.anaconda.com/pkgs/r/linux-aarch64
https://repo.anaconda.com/pkgs/r/noarch
package cache : /opt/conda/pkgs
/root/.conda/pkgs
envs directories : /opt/conda/envs
/root/.conda/envs
platform : linux-aarch64
user-agent : conda/4.12.0 requests/2.27.1 CPython/3.9.12 Linux/5.10.124-linuxkit debian/11 glibc/2.31
UID:GID : 0:0
netrc file : None
offline mode : False
\ No newline at end of file
docs/source/ch11_data_products.rst deleted 100644 → 0
View file @ 55e4b081
Data
.. code-block:: python
import re
pattern = re.compile(
r"(?P<telescope>[A-Z]+)_"
r"(?P<instrument>[A-Z]+)_"
r"(?P<project>[A-Z]+)_"
r"(?P<data_type>[A-Z]+)_"
r"(?P<exp_start>[0-9]{14})_"
r"(?P<exp_stop>[0-9]{14})_"
r"(?P<obs_id>[0-9]{11})_"
r"(?P<detector>[0-9]{2})_"
r"L(?P<level>[0-9]{1}+)_"
r"V(?P<version>[0-9]{2}+).fits"
)
mo = re.fullmatch(pattern, r"CSST_MSC_MS_SCI_20270626203558_20270626203828_100000066_01_L0_1.fits")
print(mo.groupdict())
\ No newline at end of file
docs/source/conf.py deleted 100644 → 0
View file @ 55e4b081
# Configuration file for the Sphinx documentation builder.
#
# This file only contains a selection of the most common options. For a full
# list see the documentation:
# https://www.sphinx-doc.org/en/master/usage/configuration.html
# -- Path setup --------------------------------------------------------------
# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#
import os
import sys
# . = /docs/source
# sys.path.insert(0, os.path.abspath('./csst_common/'))
# sys.path.insert(0, os.path.abspath('../../'))
sys.path.append(os.path.abspath('./csst_common/'))
sys.path.append(os.path.abspath('./packages/'))
# -- Project information -----------------------------------------------------
project = 'A Guide for CSST DAS Developers'
copyright = '2022, CSST DAS Team'
author = 'Bo Zhang'
# The full version, including alpha/beta/rc tags
release = '0.0.1beta'
# -- General configuration ---------------------------------------------------
# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = [
"nbsphinx",
"recommonmark",
"sphinx_copybutton",
"sphinx.ext.autodoc",
"sphinx.ext.napoleon",
"sphinx.ext.viewcode",
"sphinx.ext.mathjax",
]
# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']
# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
# This pattern also affects html_static_path and html_extra_path.
exclude_patterns = []
# -- Options for HTML output -------------------------------------------------
# The theme to use for HTML and HTML Help pages. See the documentation for
# a list of builtin themes.
#
html_theme = 'sphinx_rtd_theme'
# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']
# -- Support Chinese -------------------------------------------------------
# latex_engine = 'xelatex'
# latex_use_xindy = False
# latex_elements = {
# 'preamble': '\\usepackage[UTF8]{ctex}\n',
# }
# -- Copy Button --------------------------------------------------------
# copybutton_exclude = ['.linenos']
html_css_files = [
'custom.css',
]
docs/source/csst_common/example_interface.py deleted 100644 → 0
View file @ 55e4b081
import logging
import os
from typing import Union
import numpy as np
from astropy.io import fits
import joblib
from csst_common.data_manager import CsstMsDataManager
from csst_common.file_recorder import FileRecorder
from csst_common.logger import get_logger
from csst_common.status import CsstStatus
def read_image(filepath_input: str) -> np.ndarray:
""" Read image. """
return fits.getdata(filepath_input)
def process_data(data: np.ndarray) -> np.ndarray:
""" Process data. """
return np.fliplr(np.flipud(data))
# process a single image (NOT RECOMMENDED!)
def process_single_image(
filepath_input: str,
filepath_output: str,
logger: Union[None, logging.Logger] = None
) -> tuple[CsstStatus, FileRecorder]:
"""
Flip a single image.
Flip a single image with ``numpy.fliplr`` and ``numpy.flipud``.
Parameters
----------
filepath_input : str
The input filepath.
filepath_output : str
The output filepath.
logger : logging.Logger
The logger.
Returns
-------
tuple[CsstStatus, FileRecorder]
The final status.
Examples
--------
>>> process_single_image(
>>> filepath_input="input_image.fits",
>>> filepath_output="output_image.fits",
>>> logger=None
>>> )
"""
# set default logger
if logger is None:
logger = get_logger()
# get an empty file recorder
fr = FileRecorder()
# process data
logger.info("Start processing image {}".format(filepath_input))
# start processing
data = read_image(filepath_input)
data_processed = process_data(data)
np.save(filepath_output, data_processed)
# record file!
fr.add_record(filepath=filepath_output, db=True, comment="the processed image")
# this will be written into the log file
logger.info("Finish processing, result saved to {}".format(filepath_output))
# check result existence
if os.path.exists(filepath_output):
# file exists, check status and precision
if fits.getheader(filepath_output)["STT_DIST"] == 0 and \
fits.getheader(filepath_output)["PCS_DIST"] < 1e-5 and \
fits.getheader(filepath_output)["NS_DIST"] >= 8:
return CsstStatus.PERFECT, fr
else:
return CsstStatus.WARNING, fr
else:
# file doesn't exist, do your fallback solution
fits.HDUList(fits.PrimaryHDU()).writeto(filepath_output)
assert os.path.exists(filepath_output)
return CsstStatus.WARNING, fr
# process multiple images in an exposure (RECOMMENDED, at least for MBI or SLS)
# define a single job
def one_job(dm: CsstMsDataManager, detector: int):
""" Process a single image, defined for parallel processing. """
filepath_input = dm.l0_detector(detector=detector)
filepath_output = dm.l1_detector(detector=detector, post="L1_processed.fits")
# data processing
data = read_image(filepath_input)
data_processed = process_data(data)
np.save(filepath_output, data_processed)
# check result existence
if os.path.exists(filepath_output):
# file exists, check status and precision
if fits.getheader(filepath_output)["STT_DIST"] == 0 and \
fits.getheader(filepath_output)["PCS_DIST"] < 1e-5 and \
fits.getheader(filepath_output)["NS_DIST"] >= 8:
return CsstStatus.PERFECT
else:
return CsstStatus.WARNING
else:
# file doesn't exist, do your fallback solution
fits.HDUList(fits.PrimaryHDU()).writeto(filepath_output)
assert os.path.exists(filepath_output)
return CsstStatus.WARNING
# process in serial / parallel
def process_multiple_images(
dm: CsstMsDataManager,
) -> tuple[CsstStatus, FileRecorder]:
"""
Flip all images.
Flip all images in an exposure in a for-loop (serial and parallel).
Parameters
----------
dm : CsstMsDataManager
The data manager of the specified exposure.
Returns
-------
tuple[CsstStatus, FileRecorder]
The final status.
Examples
--------
>>> dm = CsstMsDataManager.quickstart(
>>> ver_sim="C5.2", dir_l1="", datatype="sls", exposure_id=100)
>>> process_multiple_images(dm)
"""
# get an empty file recorder
fr = FileRecorder()
# process data
# start processing (dm.target_detectors is a list of detector number that should be processed)
# [1/2] single-thread mode
for detector in dm.target_detectors:
# this will NOT be written into the log file
dm.logger_mod.info("Start data processing for detector {}".format(detector))
filepath_input = dm.l0_detector(detector=detector)
filepath_output = dm.l1_detector(detector=detector, post="L1_processed.fits")
data = read_image(filepath_input)
data_processed = process_data(data)
np.save(filepath_output, data_processed)
# record file!
fr.add_record(filepath=filepath_output, db=True, comment="processed file for Detector {}".format(detector))
# [2/2] multi-processing mode
dm.logger_mod.info("Starting data processing with multiprocessing ...")
status_list = joblib.Parallel(n_jobs=dm.n_jobs, backend=dm.backend)(
joblib.delayed(one_job)(dm, detector) for detector in dm.target_detectors
)
dm.logger_mod.info("Finished processing ...")
for detector in dm.target_detectors:
filepath_output = dm.l1_detector(detector=detector, post="L1_processed.fits")
fr.add_record(filepath=filepath_output, db=True, comment="processed file for Detector {}".format(detector))
# check results
assert fr.is_good()
return CsstStatus.PERFECT if all([_ == CsstStatus.PERFECT for _ in status_list]) else CsstStatus.WARNING, fr
docs/source/demo_type_annotation.ipynb deleted 100644 → 0
View file @ 55e4b081
%% Cell type:markdown id:63584611-70c6-45e4-acfb-9bda5204fda0 tags:
# How to do Python type annotation
%% Cell type:markdown id:7cc03b6e-5bc4-47e4-b42c-3a22eb11539b tags:
## built-in types
%% Cell type:code id:bfaf414a-40f8-4830-8be4-65f43a03d513 tags:
``` python
a: int = 8 # 整数型标注
b: float = 1.23 # 浮点型标注
c: bool = True # 布尔型标注
d: tuple = (1, 2) # 元组型标注
e: tuple[int, float] = (1, 2.34) # 限定类型和元素个数的元组标注
f: tuple[int, ...] = (1, 2, 3, 4, 5) # 限定类型但不定元素个数的元组型标注
g: list = [1, 2, 3] # 一般的列表型标注
h: list[int] = [1, 2, 3] # 对列表型来说不需要标元素个数，但可以明确要求元素类型
```
%% Cell type:markdown id:ba646556-0bfa-4a8f-8767-9d76e5c44592 tags:
## `typing` 模块
对于一些内建类型来说是可以直接用内建类型来标注的，例如以下两句是相等的
%% Cell type:code id:8759dc5b-01e9-4583-ab27-114ea87315db tags:
``` python
l1: list[int] = [1, 2, 3]
from typing import List
l2: List[int] = [1, 2, 3]
```
%% Cell type:markdown id:8f2203cb-8e94-406e-b445-809d04f85e26 tags:
## Union, Optional和Any
- Union表示并集，Union[float, int]表示可以是float也可以是int
- Optional表示可选，Optional[int]表示可以是None也可以是int
- Any表示任意类型
%% Cell type:code id:3b323abd-7689-4abd-aa02-049b1de7bfcb tags:
``` python
from typing import Optional, Union
x_optional: Optional[int] = None
x_union: Union[float, int] = 1.23
```
%% Cell type:markdown id:4906826b-2817-4a5b-b3ac-a993fa96b315 tags:
## numpy.typing
我们经常会用到numpy.ndarray进行科学计算，可以有以下几种标注方式:
1. 直接用np.ndarray进行标注, 此时不限定类型(dtype)
%% Cell type:code id:d93f8b14-5f88-4579-a671-1693d04ce144 tags:
``` python
import numpy as np
x1: np.ndarray = np.arange(10)
```
%% Cell type:code id:14b83562-b6c6-4873-8936-ad9d0c9c0e6f tags:
``` python
import numpy.typing as npt
x2: npt.NDArray = np.arange(10, dtype=float)
x3: npt.NDArray[np.int_] = np.arange(10, dtype=int)
x4: npt.NDArray[np.float_] = np.arange(10, dtype=float)
x5: npt.NDArray[np.float32] = np.arange(10, dtype=np.float32)
x6: npt.NDArray[np.bool_] = np.ones(10, dtype=bool)
x7: npt.NDArray[np.complex_] = np.ones(10, dtype=np.complex_)
```
%% Cell type:code id:cc6abe41-f8a1-4c9d-8deb-ba7dad66780f tags:
``` python
```
docs/source/dev_tutorial.rst deleted 100644 → 0
View file @ 55e4b081
开发指南（中文版）
=================
关于代码仓库的规划
-----------------------
CSST数据系统的所有代码都在 gitlab上：
- 1级流水线： https://csst-tb.bao.ac.cn/code/csst-l1
- msc: 主巡天
- mbi: 主巡天多色成像
- sls: 主巡天无缝光谱
- ast: 主巡天天梯测量
- mci: 多通道成像仪
- ifs: 积分场光谱仪
- cpic: 星冕仪
- hstdm: 太赫兹模块
- ooc: 在轨定标
- csst-doc: 流水线文档
- csst-proto: 流水线演示模块
- 2级流水线： https://csst-tb.bao.ac.cn/code/csst-l2
- 高级流水线： https://csst-tb.bao.ac.cn/code/csst-l3
----
- 0/1/2级数据定义: https://csst-tb.bao.ac.cn/code/csst-l1/csst_dadel
- 算法API定义： https://csst-tb.bao.ac.cn/code/csst-l1/csst_design
精简版开发指南（关于 ``git``）
------------------------------
需要学会的常用命令（Linux/MacOS命令行）
- ``git clone <url>`` 从 ``url`` 下载仓库代码到本地
- ``git branch dev`` 新建 dev 分支（如果不存在）
- ``git checkout dev`` 检录（切换到） dev 分支
- ``git add ./changes.py`` 将 ``changes.py`` 加入 git 记录
- ``git commit -m "add changes.py"`` 提交更新
- ``git push`` 提交更新
- 发起 ``merge request``
docs/source/index.rst deleted 100644 → 0
View file @ 55e4b081
.. csst_proto documentation master file, created by
sphinx-quickstart on Tue Aug 23 20:39:04 2022.
You can adapt this file completely to your liking, but it should at least
contain the root `toctree` directive.
A Guide for CSST DAS Developers
===============================
.. meta::
:description lang=cn: Automate building, versioning, and hosting of your technical documentation continuously on Read the Docs.
.. image:: https://readthedocs.org/projects/csst-proto/badge/?version=latest
:target: https://csst-proto.readthedocs.io/en/latest/?badge=latest
:alt: Documentation Status
.. toctree::
:maxdepth: 2
:caption: CONTENTS
Hi!
This is an internal guide for CSST DAS developers.
The associated package, `csst_proto`_, is the prototype of CSST L1 pipeline.
The aim of this document is to provide an example of how to build a functional
quality module for CSST DAS pipline.
Here also host the basic standards of CSST DAS pipelines as well as many details.
CSST DAS pipelines are mainly based on `Python`_.
All CSST Python code should be consistent with the PEP 8 style guide.
Developers should follow the standard `Python Packaging User Guide`_ to package
their codes.
.. _Python Packaging User Guide: https://packaging.python.org/en/latest/
.. _Python: https://www.python.org/
.. _csst_proto: https://csst-tb.bao.ac.cn/code/csst-l1/csst_proto
This guide is also highly influenced by the two below.
The guide for astropy-affiliated package development
https://docs.astropy.org/en/latest/development/docguide.html
The guide for LSST developers
https://developer.lsst.io/
----
:Author: Bo Zhang
:Institute: National Astronomical Observatories, CAS (NAOC)
:Contact: bozhang@nao.cas.cn
:Last updated: 2022-10-11
:Version: 0.0.1beta
:Copyright: CSST DAS Team
----
.. toctree::
:hidden:
:maxdepth: 2
:caption: C8 TASKS
c8_changes.rst
demo_type_annotation.ipynb
.. toctree::
:hidden:
:maxdepth: 2
:caption: TEAM
ch01_team.rst
.. toctree::
:hidden:
:maxdepth: 2
:caption: GUIDES FOR DEVELOPERS
ch02_vcs.rst
ch03_packaging.rst
ch04_codestyle.rst
ch05_preference.rst
ch06_unittest.rst
ch07_simulation.rst
ch08_csst_common.rst
.. toctree::
:hidden:
:maxdepth: 2
:caption: CODE MANAGEMENT
ch09_build.rst
ch10_integration.rst
.. toctree::
:hidden:
:maxdepth: 2
:caption: API
api/csst_proto.rst
docs/source/preference/example_joblib.py deleted 100644 → 0
View file @ 55e4b081
import time
import joblib
def f(duration=5):
""" this function will keep working for ``duration`` seconds """
a = 0
t0 = time.time()
while time.time() - t0 < duration:
a += 1
return
if __name__ == '__main__':
t_start = time.time()
joblib.Parallel(n_jobs=5, backend="loky", verbose=20)(
joblib.delayed(f)(_) for _ in [5, 5, 5, 5, 5]
)
print("Total time cost: {} sec!".format(time.time() - t_start))
docs/source/preference/example_multiprocessing.py deleted 100644 → 0
View file @ 55e4b081
import time
from multiprocessing import Pool
def f(duration=5):
""" this function will keep working for ``duration`` seconds """
a = 0
t0 = time.time()
while time.time() - t0 < duration:
a += 1
return
if __name__ == '__main__':
t_start = time.time()
# using ``with ... as ...`` clause helps avoid ``Pool.close`` after the context
with Pool(5) as p:
p.map(f, [5, 5, 5, 5, 5])
print("Total time cost: {} sec!".format(time.time() - t_start))
readthedocs.yml deleted 100644 → 0
View file @ 55e4b081
# .readthedocs.yml
# Read the Docs configuration file
# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
# Required
version: 2
build:
os: ubuntu-22.04
tools:
python: "3.9"
nodejs: "16"
# Build documentation in the docs/ directory with Sphinx
sphinx:
builder: html
configuration: docs/source/conf.py
fail_on_warning: true
# Optionally build your docs in additional formats such as PDF
formats:
- pdf
- htmlzip
# Optionally set the version of Python and requirements required to build your docs
python:
install:
- requirements: docs/requirements.txt
- method: pip
path: .
submodules:
include: []
recursive: False
search:
ranking:
api/v1/*: -1
api/v2/*: 4
ignore:
- 404.html
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment