added parallel examples and package preference

doc/source/conf.py

@@ -16,6 +16,7 @@ import sys
 # sys.path.insert(0, os.path.abspath('./csst_common/'))
 # sys.path.insert(0, os.path.abspath('../../'))
 sys.path.append(os.path.abspath('./csst_common/'))
+sys.path.append(os.path.abspath('./packages/'))
 
 
 # -- Project information -----------------------------------------------------

doc/source/index.rst

@@ -62,6 +62,7 @@ The guide for LSST developers
    vcs.rst
    packaging.rst
    codestyle.rst
+   packages/packages.rst
    unittest.rst
    simulation.rst
    csst_common/csst_common.rst

doc/source/packages/example_joblib.py

+import time
+import joblib
+
+
+def f(duration=5):
+    """ this function will keep working for ``duration`` seconds """
+    a = 0
+    t0 = time.time()
+    while time.time() - t0 < duration:
+        a += 1
+    return
+
+
+if __name__ == '__main__':
+    t_start = time.time()
+    joblib.Parallel(n_jobs=5, backend="loky", verbose=20)(
+        joblib.delayed(f)(_) for _ in [5, 5, 5, 5, 5]
+    )
+    print("Total time cost: {} sec!".format(time.time() - t_start))

doc/source/packages/example_multiprocessing.py

+import time
+from multiprocessing import Pool
+
+
+def f(duration=5):
+    """ this function will keep working for ``duration`` seconds """
+    a = 0
+    t0 = time.time()
+    while time.time() - t0 < duration:
+        a += 1
+    return
+
+
+if __name__ == '__main__':
+    t_start = time.time()
+    # using ``with ... as ...`` clause helps avoid ``Pool.close`` after the context
+    with Pool(5) as p:
+        p.map(f, [5, 5, 5, 5, 5])
+    print("Total time cost: {} sec!".format(time.time() - t_start))

doc/source/packages/packages.rst

+Package preference
+==================
+
+Initially we want our developers to following the
+`coding guidelines for astropy-affiliated packages <https://docs.astropy.org/en/latest/development/codeguide.html>`_
+as much as possible.
+A few important conventions and special cases should be outlined here.
+
+Basic preference
+----------------
+
+Several packages are favored over others if they can be used to solve the problem under study.
+Developers should use them as much as possible.
+
+
+Standard libraries
+    Python standard libraries have the highest priorities, e.g., ``os``, ``re``, etc.
+``numpy``, ``scipy``, ``matplotlib``
+    The ``BIG 3`` for Python scientific computing.
+``astropy`` and its ``astropy``-affiliated packages
+    For example, ``astropy.io.fits`` is favored over ``pyfits``.
+
+
+Parallel computing
+------------------
+
+The two packages are preferred for implementing `embarrassingly` parallel computing (without inter-communication).
+
+- ``multiprocessing``: https://docs.python.org/3/library/multiprocessing.html
+- ``joblib``: https://joblib.readthedocs.io/en/latest/
+
+.. literalinclude:: ./example_multiprocessing.py
+    :linenos:
+    :language: python
+    :caption: an example of using ``multiprocessing`` for parallel computing
+
+The output is
+
+.. code-block::
+
+    Total time cost: 5.095193147659302 sec!
+
+.. literalinclude:: ./example_joblib.py
+    :linenos:
+    :language: python
+    :caption: an example of using ``joblib`` for parallel computing
+
+The output is
+
+.. code-block::
+
+    [Parallel(n_jobs=5)]: Using backend LokyBackend with 5 concurrent workers.
+    [Parallel(n_jobs=5)]: Done   1 tasks      | elapsed:    5.2s
+    [Parallel(n_jobs=5)]: Done   2 out of   5 | elapsed:    5.2s remaining:    7.8s
+    [Parallel(n_jobs=5)]: Done   3 out of   5 | elapsed:    5.2s remaining:    3.5s
+    [Parallel(n_jobs=5)]: Done   5 out of   5 | elapsed:    5.2s remaining:    0.0s
+    [Parallel(n_jobs=5)]: Done   5 out of   5 | elapsed:    5.2s finished
+    Total time cost: 5.1958301067352295 sec!
+
+.. tip::
+    ``joblib`` is recommended for its highly concise syntax and verbose info -- do every thing within one statement.
+    ``n_jobs`` can be set to ``-1`` to use almost all CPUs, ``backend`` can be set to ``multiprocessing``
+    to use the backend built by standard library ``multiprocessing``, or ``loky`` for alleged high robustness.
+    Visit https://joblib.readthedocs.io/en/latest/ for more info and usages of ``joblib``,
+    such as the ``batch_size`` and ``verbose`` parameters.
+
+For parallel computing with inter-communication or distributed computing,
+we recommend developers to consider using ``mpi4py``: https://github.com/mpi4py/mpi4py.

doc/source/packaging.rst

@@ -155,6 +155,14 @@ and in ``flip_image.py``:
     :language: python
 
 
+Including C code
+----------------
+
+To include C code, developers should follow the guide provided by ``setuptools``:
+https://setuptools.pypa.io/en/latest/userguide/ext_modules.html.
+And also consider using https://docs.astropy.org/en/latest/development/ccython.html#building-c-or-cython-extensions.
+
+
 ``top_level_interface`` module
 ------------------------------
 This is a special requirement for all of CSST DAS packages.