diff --git a/.gitlab-ci.yml_ b/.gitlab-ci.yml_ deleted file mode 100644 index 212ca9a1603f99b4621f88d7a3deaef48e8f71b8..0000000000000000000000000000000000000000 --- a/.gitlab-ci.yml_ +++ /dev/null @@ -1,49 +0,0 @@ -# This file is a template, and might need editing before it works on your project. -# This is a sample GitLab CI/CD configuration file that should run without any modifications. -# It demonstrates a basic 3 stage CI/CD pipeline. Instead of real tests or scripts, -# it uses echo commands to simulate the pipeline execution. -# -# A pipeline is composed of independent jobs that run scripts, grouped into stages. -# Stages run in sequential order, but jobs within stages run in parallel. -# -# For more information, see: https://docs.gitlab.com/ee/ci/yaml/index.html#stages -# -# You can copy and paste this template into a new `.gitlab-ci.yml` file. -# You should not add this template to an existing `.gitlab-ci.yml` file by using the `include:` keyword. -# -# To contribute improvements to CI/CD templates, please follow the Development guide at: -# https://docs.gitlab.com/ee/development/cicd/templates.html -# This specific template is located at: -# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Getting-Started.gitlab-ci.yml - -stages: # List of stages for jobs, and their order of execution - - build - - test - -build-job: # This job runs in the build stage, which runs first. - stage: build - script: - - which ssh - - echo "Compiling the code..." - - echo "Compile complete." - -unit-test-job: # This job runs in the test stage. - stage: test # It only starts when the job in the build stage completes successfully. - script: - - echo "Running unit tests... " - - whoami - - uname -a - - pwd - - ls -a - - ls /nfsdata/share/csst_simulation_data/Cycle-5-SimuData/multipleBandsImaging/NGP_AstrometryON_shearOFF - - which python - - which pip - - python --version - - source /home/csstpipeline/anaconda3/bin/activate - - which python - - which pip - - python --version - - pip install -r requirements.txt - - sh install_local.sh - - pip show csst_proto - - pytest diff --git a/docs/Makefile b/docs/Makefile deleted file mode 100644 index d0c3cbf1020d5c292abdedf27627c6abe25e2293..0000000000000000000000000000000000000000 --- a/docs/Makefile +++ /dev/null @@ -1,20 +0,0 @@ -# Minimal makefile for Sphinx documentation -# - -# You can set these variables from the command line, and also -# from the environment for the first two. -SPHINXOPTS ?= -SPHINXBUILD ?= sphinx-build -SOURCEDIR = source -BUILDDIR = build - -# Put it first so that "make" without argument is like "make help". -help: - @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) - -.PHONY: help Makefile - -# Catch-all target: route all unknown targets to Sphinx using the new -# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). -%: Makefile - @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/apidoc.sh b/docs/apidoc.sh deleted file mode 100644 index f29cbb4e37c059c0a2f8eb8f8965958263fe1a27..0000000000000000000000000000000000000000 --- a/docs/apidoc.sh +++ /dev/null @@ -1,2 +0,0 @@ -rm -rf source/api/* -sphinx-apidoc -o source/api ../csst_proto --ext-viewcode --ext-githubpages diff --git a/docs/latexpdf.sh b/docs/latexpdf.sh deleted file mode 100644 index 479f1d5b625b9b70aa3f75135bdff525441c48e3..0000000000000000000000000000000000000000 --- a/docs/latexpdf.sh +++ /dev/null @@ -1,5 +0,0 @@ -make latexpdf LATEXMKOPTS="-silent" -cd build/latex || exit -latexmk -pdf -dvi- -ps- -silent 'aguideforcsstdasdevelopers.tex' -cd ../.. -open ./build/latex/aguideforcsstdasdevelopers.pdf \ No newline at end of file diff --git a/docs/make.bat b/docs/make.bat deleted file mode 100644 index 061f32f91b96f05f8fa2b52b2edcdcc19aa33d50..0000000000000000000000000000000000000000 --- a/docs/make.bat +++ /dev/null @@ -1,35 +0,0 @@ -@ECHO OFF - -pushd %~dp0 - -REM Command file for Sphinx documentation - -if "%SPHINXBUILD%" == "" ( - set SPHINXBUILD=sphinx-build -) -set SOURCEDIR=source -set BUILDDIR=build - -if "%1" == "" goto help - -%SPHINXBUILD% >NUL 2>NUL -if errorlevel 9009 ( - echo. - echo.The 'sphinx-build' command was not found. Make sure you have Sphinx - echo.installed, then set the SPHINXBUILD environment variable to point - echo.to the full path of the 'sphinx-build' executable. Alternatively you - echo.may add the Sphinx directory to PATH. - echo. - echo.If you don't have Sphinx installed, grab it from - echo.https://www.sphinx-doc.org/ - exit /b 1 -) - -%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% -goto end - -:help -%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% - -:end -popd diff --git a/docs/preview.sh b/docs/preview.sh deleted file mode 100755 index 3a23e690283b5e7d75a095c0dde78a9c8580bf9e..0000000000000000000000000000000000000000 --- a/docs/preview.sh +++ /dev/null @@ -1,3 +0,0 @@ - -make clean html -open ./build/html/index.html \ No newline at end of file diff --git a/docs/requirements.txt b/docs/requirements.txt deleted file mode 100644 index c3db7d349048883bbc87ca422fcddfaf7da7b2ec..0000000000000000000000000000000000000000 --- a/docs/requirements.txt +++ /dev/null @@ -1,15 +0,0 @@ -numpy==1.23.3 -scipy==1.9.1 -joblib==1.2.0 -setuptools==58.0.4 -astropy==5.1 -pycodestyle==2.9.1 -nbsphinx -#sphinx_copybutton==0.5.0 # this does not exclude linenos -git+https://github.com/executablebooks/sphinx-copybutton.git -git+https://csst-tb.bao.ac.cn/code/csst-l1/csst_common.git -git+https://csst-tb.bao.ac.cn/code/csst-dfs/csst-dfs-commons.git -git+https://csst-tb.bao.ac.cn/code/csst-dfs/csst-dfs-proto-py.git -git+https://csst-tb.bao.ac.cn/code/csst-dfs/csst-dfs-api-local.git -git+https://csst-tb.bao.ac.cn/code/csst-dfs/csst-dfs-api-cluster.git -git+https://csst-tb.bao.ac.cn/code/csst-dfs/csst-dfs-api.git diff --git a/docs/source/_static/custom.css b/docs/source/_static/custom.css deleted file mode 100644 index 0a9e3c5a5c0bbe7f312aaff684146f2e3d9b2e93..0000000000000000000000000000000000000000 --- a/docs/source/_static/custom.css +++ /dev/null @@ -1,3 +0,0 @@ -.wy-nav-content { - max-width: 80% !important; -} \ No newline at end of file diff --git a/docs/source/_static/logo-wordmark-light.svg b/docs/source/_static/logo-wordmark-light.svg deleted file mode 100644 index d0cbb66a1f376d8a4608010d834020796cf539b2..0000000000000000000000000000000000000000 --- a/docs/source/_static/logo-wordmark-light.svg +++ /dev/null @@ -1,134 +0,0 @@ - - - - - - - - - - image/svg+xml - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/docs/source/api/csst_proto.rst b/docs/source/api/csst_proto.rst deleted file mode 100644 index 2039436bc9a36a6fe00e3d8c459f9b20117db83b..0000000000000000000000000000000000000000 --- a/docs/source/api/csst_proto.rst +++ /dev/null @@ -1,11 +0,0 @@ -API -=== - -In this chapter, we show what the rendered Numpy-style docstring looks like -in sphinx documentation. - -.. automodule:: csst_proto.api - :noindex: - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/source/c8_changes.rst b/docs/source/c8_changes.rst deleted file mode 100644 index 50adf3208c88e8278af8b613a6b870e0fb87b86b..0000000000000000000000000000000000000000 --- a/docs/source/c8_changes.rst +++ /dev/null @@ -1,148 +0,0 @@ -C8主巡天流水线接口 -========================= - -Change list - - 仓库名修改： - - ``csst_ms*`` 修改为 ``csst_msc_*``, 注意修改文件夹路径和 ``setup.py`` 中的名称 - - 接口文件修改： - ``top_level_interface.py`` 文件改名为 ``api.py`` , 这个文件中用于导入每个 python package 与流水线的接口函数 - - 接口名称修改： - 每个开发模块需要把接口函数命名为 ``base_`` 开头的函数，例如 ``base_phot`` - - ``DataManager`` ``FileRecorder`` 不再使用, 底层接口都尽可能使用文件路径作为参数 - - 使用 ``csst_common.status.CsstResult`` 作为返回类型, 包含三个部分： - - 运行状态 ``status`` 必须是 ``CsstStatus`` 的三种类型之一 - - 输出文件列表 ``file_list``, 是一个 ``list``, 需要包含所有输出文件，包括临时文件 - - 额外输出 （一般不需要，除非有其他额外输出） - - 程序中禁止使用当前文件夹 ``.`` 或 ``./``，所有路径应使用绝对路径 - - 尽可能避免创建临时文件夹等操作 - - - 单元测试 - - 覆盖率合格线可以设置稍低 - - 由于代码接口定义更新，原则上单元测试应该会更加容易， - - 单元测试服务器： csstunittest@10.3.10.10 外网 -p 2000 159.226.170.52 - - 单元测试需要一个大表来统计所有模块的每个案例(文件、类、函数名)和其简单解释(~10个字) - - Jenkins上需要有一个随时更新的单元测试统计结果 包括覆盖率，测试案例列表等 - - - 接口定义修改： - ``csst_msc_mbi_distortion`` 样例代码如下 - - .. code-block:: python - - import os - - import numpy as np - import numpy.typing as npt - from astropy import table - from astropy.io import fits - from csst_common.status import CsstStatus, CsstResult - - - def read_data(file_path: str = "/path/to/data") -> np.ndarray: - return fits.getdata(file_path) - - - def _do_step_one(data: npt.NDArray) -> npt.NDArray: - reduced_data = np.fliplr(data) - return reduced_data - - - def _do_step_two(data: npt.NDArray) -> npt.NDArray: - reduced_data = np.flipud(data) - return reduced_data - - - def _do_step_three(data: npt.NDArray, rc: str = "/path/to/catalog") -> npt.NDArray: - trc = table.Table.read(rc) - return data + np.mean(trc["ra"]) + np.mean(trc["dec"]) - - - def base_msc_l1_mbi_image_distortion( - input_file: str, - output_file: str, - rc: str = "/path/to/gaia_dr3.fits", - **kwargs - ) -> CsstResult: - """your docstring here""" - # read data - data = read_data(input_file) - # do your algorithm - data = _do_step_one(data) - data = _do_step_two(data) - data = _do_step_three(data, rc=rc) - # write results - hl = fits.HDUList(fits.PrimaryHDU(data=data)) - hl.writeto(output_file, overwrite=True) - assert os.path.exists(output_file) - # construct CsstResult - result = CsstResult( - status=CsstStatus.PERFECT, - file_list=[ - output_file, - "/path/to/result1.fits", - "/path/to/result2.fits", - ], - ) - return result - - - -导入模块 ---------- - -.. code-block:: python - - # import modules - import numpy as np - import matplotlib.pyplot as plt - from astropy.io import fits - from astropy import table - - # import modules - from scipy import signal - from astropy import units as u - from astropy import constants as const - - # import classes / functions - from astropy.coordinates import SkyCoord - from scipy.interpolate import least_squares - - # DO NOT use 'import *' - - -.. 主巡天模块-接口列表 -.. ---------------------- - -.. .. code-block:: python - -.. # csst-l1/mbi/csst_msc_mbi_instrument -.. def base_instcorr(input_file, img_file, wht_file, flg_file) -> CsstResult -.. # csst-l1/mbi/csst_msc_mbi_distortion -.. def base_distortion(input_file, img_file, wht_file, flg_file): -> CsstResult -.. # csst-l1/mbi/csst_msc_mbi_position -.. def base_position_single(input_file, img_file, wht_file, flg_file): -> CsstResult -.. def base_position_multiple(input_file, img_file, wht_file, flg_file): -> CsstResult -.. # csst-l1/mbi/csst_msc_mbi_flux -.. def base_flux(input_file, img_file, wht_file, flg_file): -> CsstResult -.. # csst-l1/mbi/csst_msc_mbi_photometry -.. def base_phot(input_file, img_file, wht_file, flg_file): -> CsstResult -.. # csst-l1/sls/csst_msc_sls_instrument -.. def base_instcorr(input_file, output_file): -> CsstResult -.. # csst-l1/sls/csst_msc_sls_position -.. def base_position(input_file, output_file): -> CsstResult -.. # csst-l1/sls/csst_msc_sls_mosaic -.. def base_mosaic(input_files, output_file): -> CsstResult -.. # csst-l1/sls/csst_msc_sls_directimage -.. def base_dimg(input_file, output_file): -> CsstResult -.. # csst-l1/sls/csst_msc_sls_sky -.. def base_skybkg(input_file, output_file): -> CsstResult -.. # csst-l1/sls/csst_msc_sls_objextraction -.. def base_objext(input_file, output_file): -> CsstResult -.. # csst-l1/sls/csst_msc_sls_axe -.. def base_axe(input_file, output_file, dir_slsconf): -> CsstResult -.. # csst-l1/sls/csst_msc_sls_cde -.. def base_cde(input_file, output_file, dir_slsconf): -> CsstResult -.. # csst-l1/qc/csst_msc_qc0 -.. def base_qc0(input_file, output_file): -> CsstResult -.. # csst-l1/qc/csst_msc_sls_qc1 -.. def base_qc1(input_file, output_file): -> CsstResult - diff --git a/docs/source/ch01_team.rst b/docs/source/ch01_team.rst deleted file mode 100644 index 5f703e9fcfacc32583a3cc42b7f2c02d476a12a9..0000000000000000000000000000000000000000 --- a/docs/source/ch01_team.rst +++ /dev/null @@ -1,42 +0,0 @@ -CSST DAS -======== - -The `CSST`_ (Chinese Space Station Telescope) is a 2-meter space-based telescope -which is scheduled to be launched at December 2023. -The **CSST DAS** (Data Analysis System), led by Chao Liu, is the official team at -NAOC (National Astronomical Observatories, CAS) to build the data -reduction pipelines. - -.. _CSST: https://en.wikipedia.org/wiki/Xuntian - - -Facility resources ------------------- -Here should be a link to the CSST website - - -Timeline --------- -To fit the schedule of CSST, a software suite is underway, including - -- survey schedule -- simulation -- data pipeline - -The timeline is splitted into cycles, and each cycle spans 6 months. - -- C6 (2022.7-2022.12) - - integrate L1 pipelines -- C5 (2022.1-2022.6) - -Weekly meeting --------------- - -- 2022-08-26 (Fri) -- 2022-09-02 (Fri) -- 2022-09-09 (Fri) -- 2022-09-16 (Fri) -- 2022-09-23 (Fri) -- 2022-09-30 (Fri) -- 2022-10-14 (Fri) -- 2022-10-21 (Fri) diff --git a/docs/source/ch02_vcs.rst b/docs/source/ch02_vcs.rst deleted file mode 100644 index c5adda604cf6b509287c8f4658223c3963a4f28b..0000000000000000000000000000000000000000 --- a/docs/source/ch02_vcs.rst +++ /dev/null @@ -1,228 +0,0 @@ -Version Control -=============== - -A functional quality software needs a version control system, -which provides several advantages. - - 1. Record versions - so that you can rollback the software to any history version. - 2. Collaboration between developers - so that multiple developers can work simultaneously to enhance efficiency. - 3. Keep multiple braches - so that software can be developed in a separate branch - and it does not affect the stable (released) version. - 4. Code safety - the approve-merge mechanism, etc. - -``git`` -------- -``git`` is probably the most widely used version control system presently. - - - official website: https://git-scm.com/ - - **Pro Git** (2nd Ed.): https://git-scm.com/book/en/v2 - - -Basic usages ------------- -To initialize a repository: - -.. code-block:: bash - - git init - -To add a remote source - -.. code-block:: bash - - git remote add origin https://csst-tb.bao.ac.cn/code/csst-l1/csst_proto.git - -To show a remote source `origin` - -.. code-block:: bash - - git remote show origin - -To clone a remote repository - -.. code-block:: bash - - git clone https://csst-tb.bao.ac.cn/code/csst-l1/csst_proto.git - -To pull remote code to local - -.. code-block:: bash - - git pull origin main - -To view the git log - -.. code-block:: bash - - git log - -To view the current status - -.. code-block:: bash - - git status - -To add your revisions to version control - -.. code-block:: bash - - git add ./test.py - -To make a commit - -.. code-block:: bash - - git commit -m "make a test commit" - -To push to remote repository - -.. code-block:: bash - - git push origin main - -To reset repository to a specific commit - -.. code-block:: bash - - git reset 82cc809698b3b52327a0360eea10b88aacbadec6 --hard - -To configure your git - -.. code-block:: bash - - git config --global user.name "Your Name" - git config --global user.email "your_email@address" - -You can also store your credentials on disk via - -.. code-block:: bash - - git config --global credential.helper store - - -.. note:: - - Please DO NOT use command ``git add .`` which makes the Version Control system - track all files in the repository -- It may include many useless files such as - ``*.py~`` (temporary files generated by text editor). - - -Work flow with ``git`` ----------------------- -In CSST DAS pipeline development, each developer registers an account at the -`CSST DAS GitLab`_, which is a self self-managed `GitLab`_ instance -to host data processing pipelines at NAOC. -More specifically, the L1 Pipelines are developed in the `csst-l1`_ group. - -Developers are required to upload their code to the repository. -The repository should have three branches: - -- ``main`` branch: for a stable version -- ``dev`` branch: for development -- ``release`` branch: for code release - -After the pipeline is integrated (probably the end of C6), -a code reviewer is required to merge approved code revisions into ``main`` branch. - - -.. _CSST DAS GitLab: https://csst-tb.bao.ac.cn/code -.. _GitLab: https://about.gitlab.com/ -.. _csst-l1: https://csst-tb.bao.ac.cn/code/csst-l1 - - -Code review mechanism ---------------------- - -Starting from C7, developers of all packages should NOT directly push any code -to *main* branch which is protected. -A reviewer is needed to approve the committed modifications before -merging into the *main* branch. - -Developers should switch to a ``dev`` branch before doing anything. -Then commit code and push them to ``dev`` branch. - - -Workflow with ``dev`` branch ----------------------------- - -.. code-block:: - :caption: As a developer: - - (base) cham@MBP16 csst_proto % git branch -a # show all branch - dev - * main - remotes/origin/main - (base) cham@MBP16 csst_proto % git checkout dev # check out dev -> then you are in dev branch - M doc/source/conf.py - M doc/source/index.rst - M doc/source/integration.rst - M doc/source/vcs.rst - Switched to branch 'dev' - (base) cham@MBP16 csst_proto % git status # view code changes - On branch dev - Changes not staged for commit: - (use "git add ..." to update what will be committed) - (use "git restore ..." to discard changes in working directory) - modified: doc/source/conf.py - modified: doc/source/index.rst - modified: doc/source/integration.rst - modified: doc/source/vcs.rst - - Untracked files: - (use "git add ..." to include in what will be committed) - doc/contents.md - - no changes added to commit (use "git add" and/or "git commit -a") - (base) cham@MBP16 csst_proto % git add doc/source/*.rst doc/source/conf.py # add your code changes - (base) cham@MBP16 csst_proto % git commit -m "updated doc: author info and docker info" # commit your changes - [dev bbed3fa] updated doc: author info and docker info - 4 files changed, 73 insertions(+), 4 deletions(-) - (base) cham@MBP16 csst_proto % git push origin dev # as a developer you are not allowed to push to main branch - Enumerating objects: 15, done. - Counting objects: 100% (15/15), done. - Delta compression using up to 10 threads - Compressing objects: 100% (8/8), done. - Writing objects: 100% (8/8), 1.67 KiB | 1.67 MiB/s, done. - Total 8 (delta 6), reused 0 (delta 0), pack-reused 0 - remote: - remote: To create a merge request for dev, visit: - remote: http://10.3.10.28/code/csst-l1/csst_proto/-/merge_requests/new?merge_request%5Bsource_branch%5D=dev - remote: - To https://csst-tb.bao.ac.cn/code/csst-l1/csst_proto.git - * [new branch] dev -> dev - -.. code-block:: - :caption: As a code reviewer: - - (base) cham@MBP16 csst_proto % git checkout main # check out main branch - Switched to branch 'main' - Your branch is up to date with 'origin/main'. - (base) cham@MBP16 csst_proto % git merge dev # merge dev branch into main AFTER view changes - Updating d9f64d3..bbed3fa - Fast-forward - doc/source/conf.py | 2 +- - doc/source/index.rst | 11 +++++++++++ - doc/source/integration.rst | 50 +++++++++++++++++++++++++++++++++++++++++++++++--- - doc/source/vcs.rst | 14 ++++++++++++++ - 4 files changed, 73 insertions(+), 4 deletions(-) - (base) cham@MBP16 csst_proto % git push origin main # push to remote main branch - Total 0 (delta 0), reused 0 (delta 0), pack-reused 0 - To https://csst-tb.bao.ac.cn/code/csst-l1/csst_proto.git - d9f64d3..bbed3fa main -> main - -As a developer, you can also create a merge requrest to ask code reviewers to merge your changes on GitLab. - -First, click ``Merge requests`` tag, click `` - .. image:: vcs/merge_request.png -Second, select source branch and target branch, create a new merge request. - .. image:: vcs/pushed_to_dev.png -The code reviewers should then view the changes, and decide whether the changes should be approved. - .. image:: vcs/create_merge_request.png -If the changes are accepted, approve and merge the changes into the `main` branch. - .. image:: vcs/merge_approve.png -After the changes have been merged, code reviewers should leave comment on the changes. - .. image:: vcs/merged.png \ No newline at end of file diff --git a/docs/source/ch03_packaging.rst b/docs/source/ch03_packaging.rst deleted file mode 100644 index d6e6a60682f60fda236b928376968eaa21945ee0..0000000000000000000000000000000000000000 --- a/docs/source/ch03_packaging.rst +++ /dev/null @@ -1,213 +0,0 @@ -Python Packaging -================ -Python projects should be packaged in the standard way as shown in the -official `Python Packaging User Guide`_, and more specifically `here`_. - -.. _here: https://packaging.python.org/en/latest/tutorials/packaging-projects/ -.. _Python Packaging User Guide: https://packaging.python.org/en/latest/ - - -Project structure ------------------ -An example code structure is shown below. -Files/directories with asterisks (``*``) marks are optional. - -.. code-block:: - - csst_proto # the repository name - ├── csst_proto # the package name - │ ├── data # package associated data directory - │ ├── __init__.py # necessary file for a Python package - │ ├── demo.py # Python modules - │ ├── flip_image.py - │ ├── scratch.py - │ └── api.py # the top level interface module - ├── doc # *sphinx-based documentation directory - │ ├── build - │ ├── source - │ ├── Makefile - │ ├── apidoc.sh - │ ├── contents.md - │ ├── make.bat - │ └── preview.sh - ├── examples # *example scripts - │ ├── how_this_code_will_be_used.py - │ └── how_to_write_docstring.py - ├── tests # unit tests - │ ├── test_flip_image.py - │ └── test_other_functions.py - ├── LICENSE # license file - ├── README.md # README file (Markdown recommended) - ├── install.sh # a single-line installation script - ├── install_local.sh # local installation script - ├── readthedocs.yml # *configuration file for readthedocs - ├── requirements.txt # package dependencies - └── setup.py # setup file - -.. note:: - The ``__init__.py`` files is important, without which - the directory will not be seen as a ``Python package``. - -Python version --------------- - -Developers should use ``Python 3.9.X`` to implement algorithms. -The base docker image will be ``continuumio/anaconda3``, which uses -``Python 3.9.12``. - -Relative import ---------------- -When import a class / function within the same package, a ``relative import`` -should be used instead of ``absolute import``. -An example is below (``api.py``): - -.. code-block:: python - - from .flip_image import flip_image, read_test_image - from .demo import a_demo_function, ADemoClass - - __all__ = ["flip_image", "read_test_image", "a_demo_function", "ADemoClass"] - - -.. note:: - Note the ``.`` means it's a relative import. - -README.md ---------- - -A Markdown format text file to describe your package. - -.. literalinclude:: ../../README.md - -LICENSE -------- - -Among the many choices, we encourage our developers to use ``MIT`` LICENSE. -An example is below. - -.. literalinclude:: ../../LICENSE - - -``requirements.txt`` --------------------- -``requirements.txt`` is a text file, within which a line represents a package and its version. - -.. literalinclude:: ../../requirements.txt - :linenos: - -.. note:: - For each package, developers must specify a version. - DO NOT use ``>``, ``>=``, ``<``, ``<=`` or ``~=``, use ``==`` ONLY! - - -``setup.py`` ------------- -We use a classic format of ``setup.py`` file as shown below. -We refer readers to https://setuptools.pypa.io/en/latest/ for more information -and usages about the package `setuptools`_. - -.. _setuptools: https://setuptools.pypa.io/en/latest/ - -.. literalinclude:: ../../setup.py - :linenos: - :language: python - -.. note:: - - The ``install_requires`` keyword argument is commented because it takes no effect - when installed with ``pip install --no-deps``. - Please DO NOT use it. - -Include data ------------- - -Data files can be packaged together with code. -To include ``csst_proto/data/test_image.txt`` and ``csst_proto/data/table_data.vsb``, -set the keyword ``package_data`` in ``setup.py`` as below: - -.. code-block:: python - - ... - package_data={"": ["LICENSE", "README.md"], - "csst_proto": ["data/test_image.txt", - "data/table_data.csv" - ]}, - ... - -.. note:: - The ``""`` key represents the data files located directly in the project directory. - For files associated with package ``csst_proto``, link the data files to ``csst_proto`` key. - -.. note:: - DO NOT include any test data (e.g., CSST images) in the package. - We only encourage developers to pacakge configuration files inside. - Please keep your package *clean* and *tight*. - -To access the data, developers should use ``relative path``. -In the example below, we show the function ``read_test_image``, -an example of how to access data under ``csst_proto/data/``. -In ``__init__.py``: - -.. literalinclude:: ../../csst_proto/__init__.py - :linenos: - :language: python - -and in ``flip_image.py``: - -.. literalinclude:: ../../csst_proto/flip_image.py - :linenos: - :language: python - - -Including C code ----------------- - -To include C code, developers should follow the guide provided by ``setuptools``: -https://setuptools.pypa.io/en/latest/userguide/ext_modules.html. -And also consider using https://docs.astropy.org/en/latest/development/ccython.html#building-c-or-cython-extensions. - - -``top_level_interface`` module ------------------------------- -This is a special requirement for all of CSST DAS packages. -This module is used to store the interfaces (classes / functions) that -will be called by users. - -.. tip:: - Remember to set the ``__all__`` variable. - This should be a list containing all interfaces. - -.. note:: - All the interfaces in this module should use a complete ``Numpy``-style docstring - -- They will go through docstring format validations in Jenkins! - - -How to install package ----------------------- -We recommend the following way to install your package. - -.. code-block:: bash - - # install requirements - pip install -r requirements - # build extensions in place - python setup.py build_ext --inplace - # build source code - python setup.py sdist - # install from the source code - pip install dist/*.tar.gz --force-reinstall --no-deps - - -Sphinx-based documentation --------------------------- -This is currently regarded as an optional section. -But we still recommend developers to read a bit of the ``sphinx`` documentation. - -The sphinx homepage: - -- https://www.sphinx-doc.org/en/master/index.html - -A tutorial on how to write restructured text (.rst) files: - -- https://docutils.sourceforge.io/rst.html - diff --git a/docs/source/ch04_codestyle.rst b/docs/source/ch04_codestyle.rst deleted file mode 100644 index 6363bfb3a6f167319bd5da11dfb9e376c256a4d9..0000000000000000000000000000000000000000 --- a/docs/source/ch04_codestyle.rst +++ /dev/null @@ -1,134 +0,0 @@ -Code Style -========== - - -PEP 8 : the base Python style ------------------------------ -Python code should follow **PEP 8** - -- https://peps.python.org/pep-0008/ - -Whereas C/C++ code should follow **PEP 7** - -- https://peps.python.org/pep-0007/ - -We have a few requirements for our developers in addition to the original PEP8. -These are grabbed from the `Coding Guidelines for astropy-affiliated packages`_. - -.. _Coding Guidelines for astropy-affiliated packages: https://docs.astropy.org/en/latest/development/codeguide.html#coding-style-conventions - -- Only use 4-space indent -- ``import *`` is not allowed -- The use of automatic code formatters (e.g., Black) is strongly discouraged - - -Exceptions to PEP 8 -------------------- - -The following table summarizes all PEP 8 guidelines that are **not followed** -by the this Code Style Guide. - -E501 - Allow lines to have > 80 characters, but <=120 -E722 - Allow use of bare ``try ... except ...`` -E121, E123, E126, E133, E226, E241, E242, E704, W503, W504 and W505 - They are ignored because they are not rules unanimously accepted, - and PEP 8 does not enforce them. - -.. tip:: - Click `here`_ for error code explanations and conventions - -.. _here: https://pycodestyle.pycqa.org/en/latest/intro.html#error-codes - - -Naming conventions ------------------- - -All CSST Python source code is consistent with PEP 8 naming in the following ways: - -- class names are ``CamelCase`` with leading uppercase -- function names should be ``lower_case_with_underscore`` -- module variables used as module global constants are ``UPPERCASE_WITH_UNDERSCORES`` - - -Code style check ----------------- -Many tools can be used to check code style, such as - -- PyLint: https://pylint.pycqa.org/en/latest/index.html -- Flake8: https://flake8.pycqa.org/en/latest/index.html -- pycodestyle: https://pycodestyle.pycqa.org/en/latest/index.html - -All of them are from the PyCQA organization. -In our case, we recommend our developers to use the following command -(with pycodestyle) to check code style before committing code. - -.. code-block:: bash - - pycodestyle ./**/*.py --ignore=E121,E123,E126,E226,E24,E704,W503,W504,E501,E722 - - -Recommended IDEs ----------------- -For writing Python packages, two powerful IDEs are worth to recommend: - -- PyCharm: https://www.jetbrains.com/pycharm/ -- VSCode: https://code.visualstudio.com/ - -Both of them have plugins or built-in tools to check code style. - - -Numpy-style docstring ---------------------- -There are several popular docstring styles, namely Google, reStructuredText, and Numpy. -The CSST DAS adopts Numpydoc-style among the three. -We refer our developers to the Numpydoc official style guide for instructions on -how to write docstrings. - -- https://numpydoc.readthedocs.io/en/latest/format.html - -In particular, a complete example is available here. - -- https://numpydoc.readthedocs.io/en/latest/example.html - -In our case, we recommend the following method to check docstring - -.. code-block:: bash - - python -m numpydoc --validate {YOUR_PACKAGE}.api.{YOUR_FUNCTION/CLASS} - -For example, - -.. code-block:: bash - - python -m numpydoc --validate csst_proto.top_level_interface.flip_image - -An example with typical Numpy-style docstring is shown below - -.. literalinclude:: ../../csst_proto/demo.py - :linenos: - :language: python - -.. note:: - Numpy-style docstring uses a two-section summary for functions / classes, - including ``a short summary`` (usually a single line) right after the quote - and a following detailed ``extended summary``. - -.. note:: - The ``See Also`` section is not required for our development. - -.. note:: - Presently, Numpy style docstring check is only performed on the functions / classes - listed in the ``__all__`` variable in ``top_level_interface`` module. - -Markup languages ----------------- -Markdown and reStructrured Text are two of popular markup languages. -reStructrured Text is used in Numpydoc docstrings as well as sphinx-based documentation. -Markdown is much easier and usually used to write `README` for a package. - -- Markdown - - https://www.markdownguide.org/ -- reStructured Text - - https://www.writethedocs.org/guide/writing/reStructuredText/ diff --git a/docs/source/ch05_preference.rst b/docs/source/ch05_preference.rst deleted file mode 100644 index 5590b2e1ed39231db406839a0fb004f82a538aaf..0000000000000000000000000000000000000000 --- a/docs/source/ch05_preference.rst +++ /dev/null @@ -1,115 +0,0 @@ -Code Preference -=============== - -Initially we want our developers to following the -`coding guidelines for astropy-affiliated packages `_ -as much as possible. -A few important conventions and special cases should be outlined here. - -Package preference ------------------- - -Several packages are favored over others if they can be used to solve the problem under study. -Developers should use them as much as possible. - - -Standard libraries - Python standard libraries have the highest priorities, e.g., ``os``, ``re``, etc. -``numpy``, ``scipy``, ``matplotlib`` - The ``BIG 3`` for Python scientific computing. -``astropy`` and its ``astropy``-affiliated packages - For example, ``astropy.io.fits`` is favored over ``pyfits``. - - -Parallel computing ------------------- - -The two packages are preferred for implementing `embarrassingly` parallel computing (without inter-communication). - -- ``multiprocessing``: https://docs.python.org/3/library/multiprocessing.html -- ``joblib``: https://joblib.readthedocs.io/en/latest/ - -.. literalinclude:: preference/example_multiprocessing.py - :linenos: - :language: python - :caption: an example of using ``multiprocessing`` for parallel computing - -The output is - -.. code-block:: - - Total time cost: 5.095193147659302 sec! - -.. literalinclude:: preference/example_joblib.py - :linenos: - :language: python - :caption: an example of using ``joblib`` for parallel computing - -The output is - -.. code-block:: - - [Parallel(n_jobs=5)]: Using backend LokyBackend with 5 concurrent workers. - [Parallel(n_jobs=5)]: Done 1 tasks | elapsed: 5.2s - [Parallel(n_jobs=5)]: Done 2 out of 5 | elapsed: 5.2s remaining: 7.8s - [Parallel(n_jobs=5)]: Done 3 out of 5 | elapsed: 5.2s remaining: 3.5s - [Parallel(n_jobs=5)]: Done 5 out of 5 | elapsed: 5.2s remaining: 0.0s - [Parallel(n_jobs=5)]: Done 5 out of 5 | elapsed: 5.2s finished - Total time cost: 5.1958301067352295 sec! - -.. tip:: - ``joblib`` is recommended for its highly concise syntax and verbose info -- do every thing within one statement. - ``n_jobs`` can be set to ``-1`` to use almost all CPUs, ``backend`` can be set to ``multiprocessing`` - to use the backend built by standard library ``multiprocessing``, or ``loky`` for alleged high robustness. - Visit https://joblib.readthedocs.io/en/latest/ for more info and usages of ``joblib``, - such as the ``batch_size`` and ``verbose`` parameters. - -For parallel computing with inter-communication or distributed computing, -we recommend developers to consider using ``mpi4py``: https://github.com/mpi4py/mpi4py. - - -Global variables ----------------- - -Usage of ``global`` should be prohibited. -In most cases, variables should be kept in their default scopes. - -Using ``subprocess`` --------------------- - -``subprocess.run()`` is favored over ``subprocess.Popen()`` and ``os.system()`` as suggested in Python documentation: - -- The ``subprocess`` module allows you to spawn new processes, connect to their input/output/error pipes, - and obtain their return codes. This module intends to replace several older modules and functions: - - .. code-block:: python - - os.system - os.spawn* - -- The recommended approach to invoking ``subprocesses`` is to use the ``run()`` function for all use cases - it can handle. For more advanced use cases, the underlying ``Popen`` interface can be used directly. - - -Numpy multithreading --------------------- - -Numpy sometime automatically uses multithreading. To see if you are actually using OpenBLAS or MKL, use - -.. code-block:: python - - numpy.__config__.show() - -To set shut down this feature, use - -.. code-block:: python - - export MKL_NUM_THREADS=1 - export NUMEXPR_NUM_THREADS=1 - export OMP_NUM_THREADS=1 - export VECLIB_MAXIMUM_THREADS=1 - -.. note:: - In most cases, this automatic multithreading does not enhance the performance in practice. - Therefore, the above setting will be used in CSST pipeline globally. - diff --git a/docs/source/ch06_unittest.rst b/docs/source/ch06_unittest.rst deleted file mode 100644 index 8265287fbe8a4bcbbea1a56fb715dd78e1c2c65f..0000000000000000000000000000000000000000 --- a/docs/source/ch06_unittest.rst +++ /dev/null @@ -1,160 +0,0 @@ -Unit Tests -========== -Unit tests should be based on `unittest`_, -while `doctest`_ is not recommended for limited usage. -In the end, `pytest`_ is used to perform the unittests. -And `coverage`_ is used to assess the unit test coverage. - -Introduction to ``unittest`` ----------------------------- -Here we show an example unit test on ``csst_proto.top_level_interface.flip_image``. - -.. literalinclude:: ../../tests/test_flip_image.py - :linenos: - :language: python - - - -Run unit tests --------------- - -Unit tests should be based on the standard library ``unittest``, -while ``doctest`` is not recommended and should not be used. -The import mode we use here is ``importlib``. -Several packages should be installed before you run unit tests. -They are - -- ``pytest``: a tool to collect unit tests and run them -- ``coverage``: coverage statistics -- ``pytest-cov``: a plugin to provide ``--cov`` parameter for ``pytest`` - -To run unit tests collected from directory ``.``, and calculate coverage for source code in directory ``csst_proto``, - -.. code-block:: bash - - coverage run -m pytest --cov=csst_proto . --import-mode=importlib --cov-report=html --cov-report=term-missing - -You will see messages like below - -.. code-block:: none - - ============================ test session starts ============================ - platform darwin -- Python 3.9.7, pytest-7.1.3, pluggy-0.13.1 - rootdir: /Users/cham/CsstProjects/csst_proto - plugins: anyio-2.2.0, asdf-2.13.0, cov-4.0.0 - collected 4 items - - tests/test_demos.py . [ 25%] - tests/test_flip_image.py ... [100%] - - ---------- coverage: platform darwin, python 3.9.7-final-0 ----------- - Name Stmts Miss Cover Missing - ----------------------------------------------------------------- - csst_proto/__init__.py 3 0 100% - csst_proto/demo.py 8 3 62% 56-57, 60 - csst_proto/flip_image.py 21 0 100% - csst_proto/top_level_interface.py 3 0 100% - ----------------------------------------------------------------- - TOTAL 35 3 91% - Coverage HTML written to dir htmlcov - - ============================= 4 passed in 0.73s ============================= - - -The coverage info is shown, and a coverage report in html format will be generated in directory ``htmlcov``. - -.. _unittest: https://docs.python.org/3.9/library/unittest.html -.. _doctest: https://docs.python.org/3.9/library/doctest.html -.. _pytest: https://docs.pytest.org/en/7.1.x -.. _coverage: https://coverage.readthedocs.io/en/6.5.0/ - -Exclude files -------------- - -To exclude specific files from coverage report, write a `.coveragerc` file. - -.. code-block:: - - [run] - omit = - # omit anything in a .local directory anywhere - */.local/* - # omit everything in /usr - /usr/* - # omit this single file - utils/tirefire.py - -An example `.coveragerc` for `csst_proto` is - -.. code-block:: - - [run] - omit = - tests/* - setup.py - csst_proto/top_level_interface.py - test_codestyle.py - - -Conventions ------------ -Before write his own unit test scripts, a developer should have a glance at -the conventions on how `pytest`_ discovers unit tests (e.g, `test_*.py` or `*_test.py` files). - -For more information, click `here`_. - -.. _here: https://docs.pytest.org/en/7.1.x/explanation/goodpractices.html#test-discovery - - -Data access ------------ -For the Main Survey, i.e., the multi-band imaging (MBI) and the slit-less spectra (SLS) -developers might use two kinds of data to run their unit tests. - -Auxiliary data (data used for your unit test ONLY) - It should be put in ``/nfsdata/users/csstpipeline/L1Pipeline/unittests//`` - You can obtain this path from the environment in your unit test scirpts, - - .. code-block:: python - - import os - os.getenv("AUX_DIR") - - -Simulation data (officially released simulation data) - It should be accessed with the ``csst_common.data_manager.CsstMsDataManager``. - The ``CsstMsDataManager.quickstart`` provides an interface to initialize - an instance. See the next chapter for more details. - - Here we show an example which initializes the ``CsstMsDataManager`` with the 100th exposure of ``C5.2`` - simulation to perform a unit test for the project ``csst_ms_sls_instrument``. - - .. code-block:: python - - import os - import unittest - from csst_common.data_manager import CsstMsDataManager - from csst_ms_sls_instrument.top_level_interface import sls_instrument_pipeline - - - aux_dir = os.getenv("AUX_DIR") - out_dir = os.path.join(aux_dir, "csst_ms_sls_instrument") - - - class TestCsstMsSlsInstrument(unittest.TestCase): - def setUp(self) -> None: - self.dm = CsstMsDataManager.quickstart(ver_sim="C5.2", datatype="sls", dir_l1=out_dir, exposure_id=100) - - def test_instrument(self): - test_detector = 1 - # for l0 image path: - self.dm.l0_detector(test_detector) - # for l1 image path: - self.dm.l1_detector(detector=test_detector, post="flt.fits") - self.dm.l1_file(name="merged.fits", comment="This is a merged image.") - self.assertTrue(os.path.exists(self.dm.l0_detector(test_detector))) - - - -For developers on other instruments (CPIC, THZ, MCI and IFS), -unit tests might be performed in their local environments. diff --git a/docs/source/ch07_simulation.rst b/docs/source/ch07_simulation.rst deleted file mode 100644 index 2a5ca5b24bb72b190c910adda8eb7866e1617d5f..0000000000000000000000000000000000000000 --- a/docs/source/ch07_simulation.rst +++ /dev/null @@ -1,48 +0,0 @@ -Using Simulation Data -===================== - -Versions --------- -The current available CSST simulation data products are not versioned. -Therefore, in pipeline system, we use ``ver_sim`` keyword in code (e.g., ``CsstMsDataManager``). -The ``ver_sim`` uses a ``C{X}.{Y}`` format string, where -``X`` denotes the Cycle during which the simulation is released, -and ``Y`` denotes the serial number. -For example, ``C5.2`` represents for the second released version of simulation in Cycle 5. - - -Server details --------------- - -Server - ``dandelion``. - The ``IP``, ``Username`` and ``Password`` are available via request to ``csst_das@nao.cas.cn``. -Simulation data path - - ``C5.2``: ``/nfsdata/share/csst_simulation_data/Cycle-5-SimuData`` - - ``C3``: ``/nfsdata/share/csst_simulation_data/Cycle-3-SimuData`` - -.. note:: - Presently, the ``CsstMsDataManager`` only supports ``C5.2`` simulation. - - -Naming conventions ------------------- - -.. code-block:: python - - >>> import re - >>> pattern = re.compile( - >>> r"CSST_" - >>> r"(?P[A-Z]+)_" - >>> r"(?P[A-Z]+)_" - >>> r"(?P[A-Z]+)_" - >>> r"(?P[0-9]{14})_" - >>> r"(?P[0-9]{14})_" - >>> r"(?P[0-9]{9})_" - >>> r"(?P[0-9]{2})_" - >>> r"L(?P[0-9]+)_" - >>> r"(?P[A-Z0-9]+).fits" - >>> ) - >>> mo = re.fullmatch(pattern, r"CSST_MSC_MS_SCI_20270626203558_20270626203828_100000066_01_L0_1.fits") - >>> print(mo.groupdict()) - {'facility': 'MSC', 'project': 'MS', 'data_type': 'SCI', 't_start': '20260612100759', 't_stop': '20260612101029', 'obs_id': '100000036', 'detector': '16', 'level': '0', 'version': '1'} diff --git a/docs/source/ch08_csst_common.rst b/docs/source/ch08_csst_common.rst deleted file mode 100644 index 5c4e12489b98a7c4d9a3d21abac7e836514f335d..0000000000000000000000000000000000000000 --- a/docs/source/ch08_csst_common.rst +++ /dev/null @@ -1,328 +0,0 @@ -Using ``csst_common`` -===================== - -In this chapter, we introduce the basic usages of some IMPORTANT functions / classes in ``csst_common``. -Package source code and installation guide can be found in the link below. - -- https://csst-tb.bao.ac.cn/code/csst-l1/csst_common - -``csst_common.data_manager.CsstMsDataManager`` ----------------------------------------------- - -This class provides an interface to switch between local files and DFS. - -To initilize from local directory: - -.. code-block:: python - - from csst_common.data_manager import CsstMsDataManager - # for full basic initialization - dm = CsstMsDataManager.from_dir( - ver_sim="C5.2", # version of simulation, "C5.2" is the latest - datatype="mbi", # data type, "mbi", "sls" or "all" - dir_l0=".", # the L0/input directory - dir_l1=".", # the L1/output directory - path_aux="", # aux file paths (master bias, dark, flat) - dfs_mode=False, # if True, use DFS in pipeline modules - dfs_node='kmust', # DFS server, chosen from {"kmust", "pml"} - ) - dm.target_detectors = None # use all available detectors - - -In particular, if you are on ``dandelion`` or ``PML node``, you can use a ``quickstart`` method. - -.. code-block:: python - - # for a quick start () - dm = CsstMsDataManager.quickstart( - ver_sim='C5.2', # version of simulation, only "C5.2" is supported - datatype='mbi', # data type, "mbi", "sls" or "all" - dir_l1='.', # the L1/output directory - exposure_id=100, # the exposure ID - dfs_mode=False, # if True, use DFS in pipeline modules - dfs_node='kmust', # DFS server, chosen from {"kmust", "pml"} - ) - dm - -You will get the outputs: - -.. code-block:: - - - Data type = mbi - Valid detectors = [6, 7, 8, 9, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 22, 23, 24, 25] - Available detectors = [6, 7, 8, 9, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 22, 23, 24, 25] - Target detectors = [6, 7, 8, 9, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 22, 23, 24, 25] - dir_l0 = /nfsdata/share/csst_simulation_data/Cycle-5-SimuData/multipleBandsImaging/NGP_AstrometryON_shearOFF/MSC_0000100/ - dir_l1 = . - dfs_mode = False - dfs_node = kmust - - -.. note:: - This ``quickstart`` method is ONLY available on - ``dandelion`` or the ``PML node``. - In your local development environment, you have to use - ``CsstMsDataManager.from_dir(...)`` to initialize it. - -Attributes -^^^^^^^^^^ - -Some attributes can be accessed as below. - -.. code-block:: python - - # access L0 directory - dm.dir_l0 - # access L1 directory - dm.dir_l1 - # access dir_pcref - dm.dir_pcref - # access path_aux - dm.path_aux - # access ver_sim - dm.ver_sim - # access target detectors - dm.target_detectors - # access available detectors - dm.available_detectors - # define an L1 file (detector-specified) - dm.l1_detector(detector=6) - # define an L1 file (non-detector-specified) - dm.l1_file("flipped_image.fits") - -To get master files: - -.. code-block:: python - - # to get bias - dm.get_bias() - # to get dark - dm.get_dark() - # to get flat - dm.get_flat() - -Query Reference Catalog -^^^^^^^^^^^^^^^^^^^^^^^ - -``CsstMsDataManager`` provides a method ``query_rc`` to query reference catalog from ``DFS``. - -.. code-block:: python - - cat = dm.query_rc(ra=180, dec=0, radius=2, min_mag=0, max_mag=30) - if cat is None: - print("The query failed!") - else: - print("The query succeeded!") - - -Access code to data products ----------------------------- - -``Multi-Band Imaging`` -^^^^^^^^^^^^^^^^^^^^^^ - -========================== ===== ========================================================== ================== -Data source Level File name / access code Notes -========================== ===== ========================================================== ================== -raw 0 ``dm.l0_detector(detector=detector)`` sci image -raw 0 ``dm.l0_crs(detector=detector)`` cosmic ray image -raw 0 ``dm.l0_cat(detector=detector)`` input catalog -raw 0 ``dm.l0_log(detector=detector)`` log file -``csst_ms_mbi_instrument`` -- ``dm.l1_detector(detector=detector, post="img.fits")`` corrected image -``csst_ms_mbi_instrument`` -- ``dm.l1_detector(detector=detector, post="flg.fits")`` flag image -``csst_ms_mbi_instrument`` -- ``dm.l1_detector(detector=detector, post="wht.fits")`` weight image -``csst_ms_mbi_instrument`` -- ``dm.l1_detector(detector=detector, post="img.head")`` L0 fits header -``DFS`` -- ``dm.l1_file(name="gaia_dfs.fits")`` reference catalog -``csst_ms_mbi_distortion`` -- ``dm.l1_detector(detector=detector, post="wcs.head")`` fits head with WCS -``csst_ms_mbi_position`` -- ``dm.l1_detector(detector=detector, post="img.acat")`` photometry catalog -``csst_ms_mbi_position`` -- ``dm.l1_detector(detector=detector, post="img.rcat")`` reference catalog -``csst_ms_mbi_flux`` 1 ``dm.l1_detector(detector=detector, post="img_L1.fits")`` sci image -``csst_ms_mbi_flux`` 1 ``dm.l1_detector(detector=detector, post="flg_L1.fits")`` flag image -``csst_ms_mbi_flux`` 1 ``dm.l1_detector(detector=detector, post="wht_L1.fits")`` weight image -``csst_ms_mbi_photometry`` 1 ``dm.l1_detector(detector=detector, post="cat.fits")`` source catalog -``csst_ms_mbi_photometry`` 1 ``dm.l1_detector(detector=detector, post="psf.fits")`` source catalog -========================== ===== ========================================================== ================== - - -``Slit-Less Spectra 2D`` -^^^^^^^^^^^^^^^^^^^^^^^^ -========================== ===== =========================================================== ================== -Data source Level File name / access code Notes -========================== ===== =========================================================== ================== -raw 0 ``dm.l0_detector(detector=detector)`` sci image -raw 0 ``dm.l0_crs(detector=detector)`` cosmic ray image -raw 0 ``dm.l0_cat(detector=detector)`` input catalog -raw 0 ``dm.l0_log(detector=detector)`` log file -``csst_ms_sls_instrument`` 1 ``dm.l1_detector(detector=detector, post="L1_1_flt.fits")`` corrected image -``csst_ms_sls_position`` -- ``dm.l1_detector(detector=detector, post="L1_1.fits")`` WCS determined -========================== ===== =========================================================== ================== - - -``Slit-Less Spectra 1D`` -^^^^^^^^^^^^^^^^^^^^^^^^ - -Note that for 1D pipeline, ``dm.target_detectors`` is a list containing only one element. - -=========================== ===== =============================================================== ================== -Data source Level File name / access code Notes -=========================== ===== =============================================================== ================== -``csst_ms_sls_position`` -- ``dm.l1_detector(detector=detector, post="L1_1.fits")`` L1 SLS image -``csst_common`` -- ``dm.target_detectors[0]`` detector number -``csst_ms_sls_mosaic`` -- ``dm.l1_detector(detector=detector, post="coadd.fits")`` coadded image -``csst_ms_sls_mosaic`` -- ``dm.l1_detector(detector=detector, post="coadd.weight.fits")`` coadded weight -``csst_ms_sls_directimage`` -- ``dm.l1_detector(detector=detector, post="dir_img.fits")`` cropped image -``csst_ms_sls_directimage`` -- ``dm.l1_detector(detector=detector, post="dir_wht.fits")`` cropped weight -``csst_ms_sls_objextract`` -- ``dm.l1_detector(detector=detector, post="comb.cat")`` combined catalog -``csst_ms_sls_objextract`` -- ``dm.l1_detector(detector=detector, post="segmentation.fits")`` segmentation file -``csst_ms_sls_sky`` -- ``dm.l1_detector(detector=detector, post="L1_1_sky.fits")`` sky background -``csst_ms_sls_axe`` -- ``dm.l1_detector(detector=detector, post="L1_1_axe.fits")`` aXe spectra -``csst_ms_sls_cde`` -- ``dm.l1_detector(detector=detector, post="L1_1_cde.fits")`` CDE spectra -=========================== ===== =============================================================== ================== - - -``csst_common.file_recorder.FileRecorder`` ------------------------------------------- - -Get an empty ``FileRecorder``. -This is initially a ``list``-like object. -Use ``FileRecorder.add_record()`` to add file records. - -.. code-block:: - :linenos: - - >>> from csst_common.file_recorder import FileRecorder - >>> fr = FileRecorder() - >>> for i in range(3): - >>> fr.add_record(filepath="test{:03d}.txt".format(i), - >>> db=True, - >>> comment="Test file {:d}".format(i)) - >>> fr.pprint_all() - - - filepath db comment existence - ----------- ---- ----------- --------- - test000.txt True Test file 0 False - test001.txt True Test file 1 False - test002.txt True Test file 2 False - -``FileRecorder`` inherits from ``list``. -Therefore, if you are collecting records from subprocesses, -here is the way to combine records: - -.. code-block:: python - :linenos: - - >>> from csst_common.file_recorder import FileRecorder - >>> fr1 = FileRecorder() - >>> fr1.add_record(filepath="test1.txt", db=False, comment="test file 1") - >>> fr2 = FileRecorder() - >>> fr2.add_record(filepath="test2.txt", db=False, comment="test file 2") - >>> fr_all = FileRecorder([*fr1, *fr2]) - >>> fr_all.pprint_all() - - filepath db comment existence - --------- ----- ----------- --------- - test1.txt False test file 1 False - test2.txt False test file 2 False - - >>> fr_all.extend(fr1) - >>> fr_all.pprint_all() - - filepath db comment existence - --------- ----- ----------- --------- - test1.txt False test file 1 False - test2.txt False test file 2 False - test1.txt False test file 1 False - - -``csst_common.logger.get_logger()`` ------------------------------------ - -Get the default configured ``logging.Logger``. - - -.. code-block:: python - :linenos: - - from csst_common.logger import get_logger - logger = get_logger() - -.. warning:: - Developers should NOT use ``print`` function extensively in code. - Generally ``print`` can be replaced with ``logger.debug("msg")``. - For important information, use ``logger.info("msg")``. - For warnings, use ``logger.warning("msg")``. - For errors, use ``logger.error("msg")``. - Conf https://docs.python.org/3/library/logging.html for more information - about the usage of ``logging.Logger``. - - -``csst_common.status.CsstStatus`` ---------------------------------- - -Developers should use ``csst_common.status.CsstStatus`` to return the -status of their interfaces. - -.. code-block:: python - :linenos: - - from csst_common.status import CsstStatus - # presently 3 kinds of status are available, they are - CsstStatus.PERFECT # all images are processed smoothly - CsstStatus.WARNING # problematic - CsstStatus.ERROR # unavailable, DO NOT use this - - -An example interface --------------------- - -We recommend our developers to use the code structure used in the example interfaces, -e.g., ``process_single_image`` and ``process_multiple_images``. -The example code is shown below. - -Source code -^^^^^^^^^^^ - -.. literalinclude:: csst_common/example_interface.py - :caption: ``example_interface.py`` - :emphasize-lines: 7-11,25-29,75-88,103-116,173-175 - :linenos: - :language: python - - -Rendered ``docstring`` -^^^^^^^^^^^^^^^^^^^^^^ - -.. automodule:: example_interface - :members: - - -``csst_common.params`` ----------------------- - -See https://csst-tb.bao.ac.cn/code/csst-l1/csst_common/-/blob/main/csst_common/data/csst_params.yml. -To use the parameters, use - -.. code-block:: python - - from csst_common.params import CSST_PARAMS as CP - - - -Module Identifier ------------------ - -to be updated - -DFS interfaces --------------- - -==================================== ====================== ========================================== -Method Function Original DFS interface -==================================== ====================== ========================================== -``CsstMsDataManager.dfs_query_rc()`` query catalog from DFS ``csst_dfs_api.common.catalog.CatalogApi`` -``CsstMsDataManager.dfs_query_l0()`` query L0 data from DFS -- -``CsstMsDataManager.dfs_query_l1()`` query L1 data from DFS -- -``CsstMsDataManager.dfs_push_l1()`` push L1 data to DFS ``csst_dfs_api.msc.level1.Level1DataApi`` -==================================== ====================== ========================================== \ No newline at end of file diff --git a/docs/source/ch09_build.rst b/docs/source/ch09_build.rst deleted file mode 100644 index 99a08802e48fbb04399f0081867ea0f0da81bc33..0000000000000000000000000000000000000000 --- a/docs/source/ch09_build.rst +++ /dev/null @@ -1,118 +0,0 @@ -Build and Test -============== - -A push to git repository will trigger an automatic build and test on the Jenkins platform. -Then a notification email from ``csst_das@nao.cas.cn`` is composed, -attached with a ``build log`` and a ``coverage report``, -is sent to relavant developers and code reviewers. - - -``build status`` explained --------------------------- - -The ``build status`` is in the subject of the notification email. -Here are the explanations of them. - -Successful - Congratulations! Your code passes all tests. -Fixed - Although last build fails, this build is successful. -Failure - Current build fails in at least one test. -Still Failing - Current build fails in at least one test. And last build fails. - - -``build log`` explained ------------------------ - -The build log includes 6 sections. - -[1/6] Check on versions of requirements - The status will be non-zero if any of your requirements is behind other developers. -[2/6] Check on installation of requirements - The status will be non-zero if ``pip install -r requirements.txt`` exit with non-zero status. -[3/6] Check on installation of package - The status will be non-zero if the following code are unsuccessfully executed - - .. code-block:: bash - - rm -rf dist # remove dist directory if exists - python setup.py build_ext --inplace # build extension - python setup.py sdist # build source code - pip install dist/*.tar.gz --force-reinstall --no-deps # install package without dependencies - -[4/6] Check on import of interfaces (``top_level_interface``) - The status will be non-zero if the following code are unsuccessfully executed - - .. code-block:: python - - from .top_level_interface import * - -[5/6] Run unit tests and coverage - The status will be non-zero if the following code are unsuccessfully executed - - .. code-block:: bash - - coverage run -m pytest --import-mode=importlib - -[6/6] ``Numpydoc`` validation of interfaces - The status will be non-zero if the following code are unsuccessfully executed - - .. code-block:: bash - - python -m numpydoc --validate {YOUR_PACKAGE}.top_level_interface.{YOUR_FUNCTION/CLASS} - - -An example of test summary is at the end of build log (the attached ``build.zip``). -If all checks/tests/validations passed, you will see a section as below. - -.. code-block:: none - - ===================== TEST SUMMARY ===================== - Test passed! - Check requirements: 0 - Install requirements: 0 - Install package: 0 - Importability status: 0 - Unit test status: 0 - Numpydoc status: 0 - - -Unit test coverage report -------------------------- - -The ``build log`` also contains a section of unit test coverage report. -An example is below. - -.. code-block:: none - - ---------- coverage: platform darwin, python 3.9.7-final-0 ----------- - Name Stmts Miss Cover Missing - ----------------------------------------------------------------- - csst_proto/__init__.py 3 0 100% - csst_proto/demo.py 8 3 62% 56-57, 60 - csst_proto/flip_image.py 21 0 100% - csst_proto/top_level_interface.py 3 0 100% - ----------------------------------------------------------------- - TOTAL 35 3 91% - - -This means the overall unit test coverage is 91%. -The missing terms are also shown in the last column. -You can also find a detailed coverage report in ``html`` format in the attached ``htmlcov.zip``. - - -Caveats -------- -The Numpy style docstring check is only for the packages listed below. - -- csst_proto -- csst_cicd -- csst_ms_mbi_instrument -- csst_ms_mbi_distortion -- csst_ms_mbi_position -- csst_ms_mbi_flux -- csst_ms_mbi_photometry - -We welcome other packages participate in. diff --git a/docs/source/ch10_integration.rst b/docs/source/ch10_integration.rst deleted file mode 100644 index 84a9750b107a0e48c38b0f2122451c010bbdec78..0000000000000000000000000000000000000000 --- a/docs/source/ch10_integration.rst +++ /dev/null @@ -1,57 +0,0 @@ -Continuous Integration -====================== - -Using ``Jenkins`` ------------------ - -to be updated - - -Base ``docker`` image ---------------------- - -The CSST pipeline will be dockerized from the base image ``continuumio/anaconda3``: - - - https://hub.docker.com/r/continuumio/anaconda3 - -The basic information is shown below - -.. code-block:: - - # cat /etc/issue - Debian GNU/Linux 11 \n \l - - # which python - /opt/conda/bin/python - # which conda - /opt/conda/bin/conda - # which pip - /opt/conda/bin/pip - # conda info - - active environment : None - user config file : /root/.condarc - populated config files : - conda version : 4.12.0 - conda-build version : 3.21.8 - python version : 3.9.12.final.0 - virtual packages : __linux=5.10.124=0 - __glibc=2.31=0 - __unix=0=0 - __archspec=1=aarch64 - base environment : /opt/conda (writable) - conda av data dir : /opt/conda/etc/conda - conda av metadata url : None - channel URLs : https://repo.anaconda.com/pkgs/main/linux-aarch64 - https://repo.anaconda.com/pkgs/main/noarch - https://repo.anaconda.com/pkgs/r/linux-aarch64 - https://repo.anaconda.com/pkgs/r/noarch - package cache : /opt/conda/pkgs - /root/.conda/pkgs - envs directories : /opt/conda/envs - /root/.conda/envs - platform : linux-aarch64 - user-agent : conda/4.12.0 requests/2.27.1 CPython/3.9.12 Linux/5.10.124-linuxkit debian/11 glibc/2.31 - UID:GID : 0:0 - netrc file : None - offline mode : False \ No newline at end of file diff --git a/docs/source/ch11_data_products.rst b/docs/source/ch11_data_products.rst deleted file mode 100644 index f761d148f2cbf2fa389e094ab540ba9b23c26039..0000000000000000000000000000000000000000 --- a/docs/source/ch11_data_products.rst +++ /dev/null @@ -1,19 +0,0 @@ -Data - -.. code-block:: python - - import re - pattern = re.compile( - r"(?P[A-Z]+)_" - r"(?P[A-Z]+)_" - r"(?P[A-Z]+)_" - r"(?P[A-Z]+)_" - r"(?P[0-9]{14})_" - r"(?P[0-9]{14})_" - r"(?P[0-9]{11})_" - r"(?P[0-9]{2})_" - r"L(?P[0-9]{1}+)_" - r"V(?P[0-9]{2}+).fits" - ) - mo = re.fullmatch(pattern, r"CSST_MSC_MS_SCI_20270626203558_20270626203828_100000066_01_L0_1.fits") - print(mo.groupdict()) \ No newline at end of file diff --git a/docs/source/conf.py b/docs/source/conf.py deleted file mode 100644 index bb9fdaec1a9929627ac3b5cfd61b6b70fdb0095c..0000000000000000000000000000000000000000 --- a/docs/source/conf.py +++ /dev/null @@ -1,80 +0,0 @@ -# Configuration file for the Sphinx documentation builder. -# -# This file only contains a selection of the most common options. For a full -# list see the documentation: -# https://www.sphinx-doc.org/en/master/usage/configuration.html - -# -- Path setup -------------------------------------------------------------- - -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. -# -import os -import sys -# . = /docs/source -# sys.path.insert(0, os.path.abspath('./csst_common/')) -# sys.path.insert(0, os.path.abspath('../../')) -sys.path.append(os.path.abspath('./csst_common/')) -sys.path.append(os.path.abspath('./packages/')) - - -# -- Project information ----------------------------------------------------- - -project = 'A Guide for CSST DAS Developers' -copyright = '2022, CSST DAS Team' -author = 'Bo Zhang' - -# The full version, including alpha/beta/rc tags -release = '0.0.1beta' - - -# -- General configuration --------------------------------------------------- - -# Add any Sphinx extension module names here, as strings. They can be -# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom -# ones. -extensions = [ - "nbsphinx", - "recommonmark", - "sphinx_copybutton", - "sphinx.ext.autodoc", - "sphinx.ext.napoleon", - "sphinx.ext.viewcode", - "sphinx.ext.mathjax", -] - -# Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] - -# List of patterns, relative to source directory, that match files and -# directories to ignore when looking for source files. -# This pattern also affects html_static_path and html_extra_path. -exclude_patterns = [] - - -# -- Options for HTML output ------------------------------------------------- - -# The theme to use for HTML and HTML Help pages. See the documentation for -# a list of builtin themes. -# -html_theme = 'sphinx_rtd_theme' - -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] - -# -- Support Chinese ------------------------------------------------------- -# latex_engine = 'xelatex' -# latex_use_xindy = False -# latex_elements = { -# 'preamble': '\\usepackage[UTF8]{ctex}\n', -# } - -# -- Copy Button -------------------------------------------------------- -# copybutton_exclude = ['.linenos'] - -html_css_files = [ - 'custom.css', -] diff --git a/docs/source/csst_common/example_interface.py b/docs/source/csst_common/example_interface.py deleted file mode 100644 index 2e4d73151d0fc699c9bb48cfdcece1075441cbdd..0000000000000000000000000000000000000000 --- a/docs/source/csst_common/example_interface.py +++ /dev/null @@ -1,175 +0,0 @@ -import logging -import os -from typing import Union - -import numpy as np -from astropy.io import fits -import joblib -from csst_common.data_manager import CsstMsDataManager -from csst_common.file_recorder import FileRecorder -from csst_common.logger import get_logger -from csst_common.status import CsstStatus - - -def read_image(filepath_input: str) -> np.ndarray: - """ Read image. """ - return fits.getdata(filepath_input) - - -def process_data(data: np.ndarray) -> np.ndarray: - """ Process data. """ - return np.fliplr(np.flipud(data)) - - -# process a single image (NOT RECOMMENDED!) -def process_single_image( - filepath_input: str, - filepath_output: str, - logger: Union[None, logging.Logger] = None -) -> tuple[CsstStatus, FileRecorder]: - """ - Flip a single image. - - Flip a single image with ``numpy.fliplr`` and ``numpy.flipud``. - - Parameters - ---------- - filepath_input : str - The input filepath. - filepath_output : str - The output filepath. - logger : logging.Logger - The logger. - - Returns - ------- - tuple[CsstStatus, FileRecorder] - The final status. - - Examples - -------- - >>> process_single_image( - >>> filepath_input="input_image.fits", - >>> filepath_output="output_image.fits", - >>> logger=None - >>> ) - """ - # set default logger - if logger is None: - logger = get_logger() - - # get an empty file recorder - fr = FileRecorder() - - # process data - logger.info("Start processing image {}".format(filepath_input)) - # start processing - data = read_image(filepath_input) - data_processed = process_data(data) - np.save(filepath_output, data_processed) - # record file! - fr.add_record(filepath=filepath_output, db=True, comment="the processed image") - # this will be written into the log file - logger.info("Finish processing, result saved to {}".format(filepath_output)) - - # check result existence - if os.path.exists(filepath_output): - # file exists, check status and precision - if fits.getheader(filepath_output)["STT_DIST"] == 0 and \ - fits.getheader(filepath_output)["PCS_DIST"] < 1e-5 and \ - fits.getheader(filepath_output)["NS_DIST"] >= 8: - return CsstStatus.PERFECT, fr - else: - return CsstStatus.WARNING, fr - else: - # file doesn't exist, do your fallback solution - fits.HDUList(fits.PrimaryHDU()).writeto(filepath_output) - assert os.path.exists(filepath_output) - return CsstStatus.WARNING, fr - - -# process multiple images in an exposure (RECOMMENDED, at least for MBI or SLS) -# define a single job -def one_job(dm: CsstMsDataManager, detector: int): - """ Process a single image, defined for parallel processing. """ - filepath_input = dm.l0_detector(detector=detector) - filepath_output = dm.l1_detector(detector=detector, post="L1_processed.fits") - - # data processing - data = read_image(filepath_input) - data_processed = process_data(data) - np.save(filepath_output, data_processed) - - # check result existence - if os.path.exists(filepath_output): - # file exists, check status and precision - if fits.getheader(filepath_output)["STT_DIST"] == 0 and \ - fits.getheader(filepath_output)["PCS_DIST"] < 1e-5 and \ - fits.getheader(filepath_output)["NS_DIST"] >= 8: - return CsstStatus.PERFECT - else: - return CsstStatus.WARNING - else: - # file doesn't exist, do your fallback solution - fits.HDUList(fits.PrimaryHDU()).writeto(filepath_output) - assert os.path.exists(filepath_output) - return CsstStatus.WARNING - - -# process in serial / parallel -def process_multiple_images( - dm: CsstMsDataManager, -) -> tuple[CsstStatus, FileRecorder]: - """ - Flip all images. - - Flip all images in an exposure in a for-loop (serial and parallel). - - Parameters - ---------- - dm : CsstMsDataManager - The data manager of the specified exposure. - - Returns - ------- - tuple[CsstStatus, FileRecorder] - The final status. - - Examples - -------- - >>> dm = CsstMsDataManager.quickstart( - >>> ver_sim="C5.2", dir_l1="", datatype="sls", exposure_id=100) - >>> process_multiple_images(dm) - """ - - # get an empty file recorder - fr = FileRecorder() - - # process data - # start processing (dm.target_detectors is a list of detector number that should be processed) - - # [1/2] single-thread mode - for detector in dm.target_detectors: - # this will NOT be written into the log file - dm.logger_mod.info("Start data processing for detector {}".format(detector)) - filepath_input = dm.l0_detector(detector=detector) - filepath_output = dm.l1_detector(detector=detector, post="L1_processed.fits") - data = read_image(filepath_input) - data_processed = process_data(data) - np.save(filepath_output, data_processed) - # record file! - fr.add_record(filepath=filepath_output, db=True, comment="processed file for Detector {}".format(detector)) - - # [2/2] multi-processing mode - dm.logger_mod.info("Starting data processing with multiprocessing ...") - status_list = joblib.Parallel(n_jobs=dm.n_jobs, backend=dm.backend)( - joblib.delayed(one_job)(dm, detector) for detector in dm.target_detectors - ) - dm.logger_mod.info("Finished processing ...") - for detector in dm.target_detectors: - filepath_output = dm.l1_detector(detector=detector, post="L1_processed.fits") - fr.add_record(filepath=filepath_output, db=True, comment="processed file for Detector {}".format(detector)) - - # check results - assert fr.is_good() - return CsstStatus.PERFECT if all([_ == CsstStatus.PERFECT for _ in status_list]) else CsstStatus.WARNING, fr diff --git a/docs/source/demo_type_annotation.ipynb b/docs/source/demo_type_annotation.ipynb deleted file mode 100644 index 51ba38ff5833283061802ebe91d376c67492b0af..0000000000000000000000000000000000000000 --- a/docs/source/demo_type_annotation.ipynb +++ /dev/null @@ -1,153 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "63584611-70c6-45e4-acfb-9bda5204fda0", - "metadata": {}, - "source": [ - "# How to do Python type annotation" - ] - }, - { - "cell_type": "markdown", - "id": "7cc03b6e-5bc4-47e4-b42c-3a22eb11539b", - "metadata": {}, - "source": [ - "## built-in types" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "id": "bfaf414a-40f8-4830-8be4-65f43a03d513", - "metadata": {}, - "outputs": [], - "source": [ - "a: int = 8 # 整数型标注\n", - "b: float = 1.23 # 浮点型标注\n", - "c: bool = True # 布尔型标注\n", - "d: tuple = (1, 2) # 元组型标注\n", - "e: tuple[int, float] = (1, 2.34) # 限定类型和元素个数的元组标注\n", - "f: tuple[int, ...] = (1, 2, 3, 4, 5) # 限定类型但不定元素个数的元组型标注\n", - "g: list = [1, 2, 3] # 一般的列表型标注\n", - "h: list[int] = [1, 2, 3] # 对列表型来说不需要标元素个数，但可以明确要求元素类型" - ] - }, - { - "cell_type": "markdown", - "id": "ba646556-0bfa-4a8f-8767-9d76e5c44592", - "metadata": {}, - "source": [ - "## `typing` 模块\n", - "对于一些内建类型来说是可以直接用内建类型来标注的，例如以下两句是相等的" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "8759dc5b-01e9-4583-ab27-114ea87315db", - "metadata": {}, - "outputs": [], - "source": [ - "l1: list[int] = [1, 2, 3]\n", - "\n", - "from typing import List\n", - "l2: List[int] = [1, 2, 3]" - ] - }, - { - "cell_type": "markdown", - "id": "8f2203cb-8e94-406e-b445-809d04f85e26", - "metadata": {}, - "source": [ - "## Union, Optional和Any\n", - "\n", - "- Union表示并集，Union[float, int]表示可以是float也可以是int\n", - "- Optional表示可选，Optional[int]表示可以是None也可以是int\n", - "- Any表示任意类型" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "3b323abd-7689-4abd-aa02-049b1de7bfcb", - "metadata": {}, - "outputs": [], - "source": [ - "from typing import Optional, Union\n", - "\n", - "x_optional: Optional[int] = None\n", - "x_union: Union[float, int] = 1.23" - ] - }, - { - "cell_type": "markdown", - "id": "4906826b-2817-4a5b-b3ac-a993fa96b315", - "metadata": {}, - "source": [ - "## numpy.typing\n", - "我们经常会用到numpy.ndarray进行科学计算，可以有以下几种标注方式:\n", - "\n", - "1. 直接用np.ndarray进行标注, 此时不限定类型(dtype)" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "id": "d93f8b14-5f88-4579-a671-1693d04ce144", - "metadata": {}, - "outputs": [], - "source": [ - "import numpy as np\n", - "\n", - "x1: np.ndarray = np.arange(10)" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "id": "14b83562-b6c6-4873-8936-ad9d0c9c0e6f", - "metadata": {}, - "outputs": [], - "source": [ - "import numpy.typing as npt\n", - "\n", - "x2: npt.NDArray = np.arange(10, dtype=float)\n", - "x3: npt.NDArray[np.int_] = np.arange(10, dtype=int)\n", - "x4: npt.NDArray[np.float_] = np.arange(10, dtype=float)\n", - "x5: npt.NDArray[np.float32] = np.arange(10, dtype=np.float32)\n", - "x6: npt.NDArray[np.bool_] = np.ones(10, dtype=bool)\n", - "x7: npt.NDArray[np.complex_] = np.ones(10, dtype=np.complex_)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "cc6abe41-f8a1-4c9d-8deb-ba7dad66780f", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.4" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/docs/source/dev_tutorial.rst b/docs/source/dev_tutorial.rst deleted file mode 100644 index aa6cff1358dbdedf0eea519d9911e85d3b36b709..0000000000000000000000000000000000000000 --- a/docs/source/dev_tutorial.rst +++ /dev/null @@ -1,50 +0,0 @@ -开发指南（中文版） -================= - - -关于代码仓库的规划 ------------------------ - -CSST数据系统的所有代码都在 gitlab上： - - - 1级流水线： https://csst-tb.bao.ac.cn/code/csst-l1 - - msc: 主巡天 - - mbi: 主巡天多色成像 - - sls: 主巡天无缝光谱 - - ast: 主巡天天梯测量 - - mci: 多通道成像仪 - - ifs: 积分场光谱仪 - - cpic: 星冕仪 - - hstdm: 太赫兹模块 - - ooc: 在轨定标 - - csst-doc: 流水线文档 - - csst-proto: 流水线演示模块 - - 2级流水线： https://csst-tb.bao.ac.cn/code/csst-l2 - - 高级流水线： https://csst-tb.bao.ac.cn/code/csst-l3 - ----- - - - 0/1/2级数据定义: https://csst-tb.bao.ac.cn/code/csst-l1/csst_dadel - - 算法API定义： https://csst-tb.bao.ac.cn/code/csst-l1/csst_design - - - - -精简版开发指南（关于 ``git``） ------------------------------- - -需要学会的常用命令（Linux/MacOS命令行） - - - ``git clone `` 从 ``url`` 下载仓库代码到本地 - - ``git branch dev`` 新建 dev 分支（如果不存在） - - ``git checkout dev`` 检录（切换到） dev 分支 - - ``git add ./changes.py`` 将 ``changes.py`` 加入 git 记录 - - ``git commit -m "add changes.py"`` 提交更新 - - ``git push`` 提交更新 - - 发起 ``merge request`` - - - - - - diff --git a/docs/source/index.rst b/docs/source/index.rst deleted file mode 100644 index 8bc0a40e3acdde26ca29995d514ca490ce35ab60..0000000000000000000000000000000000000000 --- a/docs/source/index.rst +++ /dev/null @@ -1,105 +0,0 @@ -.. csst_proto documentation master file, created by - sphinx-quickstart on Tue Aug 23 20:39:04 2022. - You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. - -A Guide for CSST DAS Developers -=============================== - -.. meta:: - :description lang=cn: Automate building, versioning, and hosting of your technical documentation continuously on Read the Docs. - - -.. image:: https://readthedocs.org/projects/csst-proto/badge/?version=latest - :target: https://csst-proto.readthedocs.io/en/latest/?badge=latest - :alt: Documentation Status - - -.. toctree:: - :maxdepth: 2 - :caption: CONTENTS - - -Hi! -This is an internal guide for CSST DAS developers. -The associated package, `csst_proto`_, is the prototype of CSST L1 pipeline. -The aim of this document is to provide an example of how to build a functional -quality module for CSST DAS pipline. -Here also host the basic standards of CSST DAS pipelines as well as many details. - -CSST DAS pipelines are mainly based on `Python`_. -All CSST Python code should be consistent with the PEP 8 style guide. -Developers should follow the standard `Python Packaging User Guide`_ to package -their codes. - -.. _Python Packaging User Guide: https://packaging.python.org/en/latest/ -.. _Python: https://www.python.org/ -.. _csst_proto: https://csst-tb.bao.ac.cn/code/csst-l1/csst_proto - - -This guide is also highly influenced by the two below. - -The guide for astropy-affiliated package development - https://docs.astropy.org/en/latest/development/docguide.html - -The guide for LSST developers - https://developer.lsst.io/ - ----- - -:Author: Bo Zhang -:Institute: National Astronomical Observatories, CAS (NAOC) -:Contact: bozhang@nao.cas.cn -:Last updated: 2022-10-11 -:Version: 0.0.1beta -:Copyright: CSST DAS Team - ----- - - -.. toctree:: - :hidden: - :maxdepth: 2 - :caption: C8 TASKS - - c8_changes.rst - demo_type_annotation.ipynb - - -.. toctree:: - :hidden: - :maxdepth: 2 - :caption: TEAM - - ch01_team.rst - - -.. toctree:: - :hidden: - :maxdepth: 2 - :caption: GUIDES FOR DEVELOPERS - - ch02_vcs.rst - ch03_packaging.rst - ch04_codestyle.rst - ch05_preference.rst - ch06_unittest.rst - ch07_simulation.rst - ch08_csst_common.rst - - -.. toctree:: - :hidden: - :maxdepth: 2 - :caption: CODE MANAGEMENT - - ch09_build.rst - ch10_integration.rst - - -.. toctree:: - :hidden: - :maxdepth: 2 - :caption: API - - api/csst_proto.rst diff --git a/docs/source/preference/example_joblib.py b/docs/source/preference/example_joblib.py deleted file mode 100644 index 6cc785ee3560da4d982ecc753fae74e271229866..0000000000000000000000000000000000000000 --- a/docs/source/preference/example_joblib.py +++ /dev/null @@ -1,19 +0,0 @@ -import time -import joblib - - -def f(duration=5): - """ this function will keep working for ``duration`` seconds """ - a = 0 - t0 = time.time() - while time.time() - t0 < duration: - a += 1 - return - - -if __name__ == '__main__': - t_start = time.time() - joblib.Parallel(n_jobs=5, backend="loky", verbose=20)( - joblib.delayed(f)(_) for _ in [5, 5, 5, 5, 5] - ) - print("Total time cost: {} sec!".format(time.time() - t_start)) diff --git a/docs/source/preference/example_multiprocessing.py b/docs/source/preference/example_multiprocessing.py deleted file mode 100644 index ff7f96c8d22cc2e2233eafc075a30efe72f8325a..0000000000000000000000000000000000000000 --- a/docs/source/preference/example_multiprocessing.py +++ /dev/null @@ -1,19 +0,0 @@ -import time -from multiprocessing import Pool - - -def f(duration=5): - """ this function will keep working for ``duration`` seconds """ - a = 0 - t0 = time.time() - while time.time() - t0 < duration: - a += 1 - return - - -if __name__ == '__main__': - t_start = time.time() - # using ``with ... as ...`` clause helps avoid ``Pool.close`` after the context - with Pool(5) as p: - p.map(f, [5, 5, 5, 5, 5]) - print("Total time cost: {} sec!".format(time.time() - t_start)) diff --git a/docs/source/vcs/create_merge_request.png b/docs/source/vcs/create_merge_request.png deleted file mode 100644 index 9c32c61517e4da0f8b0211b2e679bfe909193ddc..0000000000000000000000000000000000000000 Binary files a/docs/source/vcs/create_merge_request.png and /dev/null differ diff --git a/docs/source/vcs/merge_approve.png b/docs/source/vcs/merge_approve.png deleted file mode 100644 index 41035c854efbc60d1e4ef0d594cf2737620ccaf5..0000000000000000000000000000000000000000 Binary files a/docs/source/vcs/merge_approve.png and /dev/null differ diff --git a/docs/source/vcs/merge_request.png b/docs/source/vcs/merge_request.png deleted file mode 100644 index 45575e6a5f6c4a1b7154283ac53240107265c93a..0000000000000000000000000000000000000000 Binary files a/docs/source/vcs/merge_request.png and /dev/null differ diff --git a/docs/source/vcs/merged.png b/docs/source/vcs/merged.png deleted file mode 100644 index 994066af125d70031c31ed63d4e1546e42ca01f0..0000000000000000000000000000000000000000 Binary files a/docs/source/vcs/merged.png and /dev/null differ diff --git a/docs/source/vcs/pushed_to_dev.png b/docs/source/vcs/pushed_to_dev.png deleted file mode 100644 index fc0694ab0e1f69ed91a0ffbc65ab9f903c440053..0000000000000000000000000000000000000000 Binary files a/docs/source/vcs/pushed_to_dev.png and /dev/null differ diff --git a/readthedocs.yml b/readthedocs.yml deleted file mode 100644 index f8b6546cec3e8228f338771896d897beb1cf1720..0000000000000000000000000000000000000000 --- a/readthedocs.yml +++ /dev/null @@ -1,41 +0,0 @@ -# .readthedocs.yml -# Read the Docs configuration file -# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details - -# Required -version: 2 - -build: - os: ubuntu-22.04 - tools: - python: "3.9" - nodejs: "16" - -# Build documentation in the docs/ directory with Sphinx -sphinx: - builder: html - configuration: docs/source/conf.py - fail_on_warning: true - -# Optionally build your docs in additional formats such as PDF -formats: - - pdf - - htmlzip - -# Optionally set the version of Python and requirements required to build your docs -python: - install: - - requirements: docs/requirements.txt - - method: pip - path: . - -submodules: - include: [] - recursive: False - -search: - ranking: - api/v1/*: -1 - api/v2/*: 4 - ignore: - - 404.html