diff --git a/doc/source/conf.py b/doc/source/conf.py index 1f7e4def67c2500f804ee1539b7c7995e7b32f4a..81ab2514da47bdc39f60daf9d70e95a72926b59b 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -16,6 +16,7 @@ import sys # sys.path.insert(0, os.path.abspath('./csst_common/')) # sys.path.insert(0, os.path.abspath('../../')) sys.path.append(os.path.abspath('./csst_common/')) +sys.path.append(os.path.abspath('./packages/')) # -- Project information ----------------------------------------------------- diff --git a/doc/source/index.rst b/doc/source/index.rst index 97e134946cc25d789cf9792a57df6642717d9ea6..9ec475320e5b969c7aab743d55da3dfcbff01c1b 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -62,6 +62,7 @@ The guide for LSST developers vcs.rst packaging.rst codestyle.rst + packages/packages.rst unittest.rst simulation.rst csst_common/csst_common.rst diff --git a/doc/source/packages/example_joblib.py b/doc/source/packages/example_joblib.py new file mode 100644 index 0000000000000000000000000000000000000000..6cc785ee3560da4d982ecc753fae74e271229866 --- /dev/null +++ b/doc/source/packages/example_joblib.py @@ -0,0 +1,19 @@ +import time +import joblib + + +def f(duration=5): + """ this function will keep working for ``duration`` seconds """ + a = 0 + t0 = time.time() + while time.time() - t0 < duration: + a += 1 + return + + +if __name__ == '__main__': + t_start = time.time() + joblib.Parallel(n_jobs=5, backend="loky", verbose=20)( + joblib.delayed(f)(_) for _ in [5, 5, 5, 5, 5] + ) + print("Total time cost: {} sec!".format(time.time() - t_start)) diff --git a/doc/source/packages/example_multiprocessing.py b/doc/source/packages/example_multiprocessing.py new file mode 100644 index 0000000000000000000000000000000000000000..ff7f96c8d22cc2e2233eafc075a30efe72f8325a --- /dev/null +++ b/doc/source/packages/example_multiprocessing.py @@ -0,0 +1,19 @@ +import time +from multiprocessing import Pool + + +def f(duration=5): + """ this function will keep working for ``duration`` seconds """ + a = 0 + t0 = time.time() + while time.time() - t0 < duration: + a += 1 + return + + +if __name__ == '__main__': + t_start = time.time() + # using ``with ... as ...`` clause helps avoid ``Pool.close`` after the context + with Pool(5) as p: + p.map(f, [5, 5, 5, 5, 5]) + print("Total time cost: {} sec!".format(time.time() - t_start)) diff --git a/doc/source/packages/packages.rst b/doc/source/packages/packages.rst new file mode 100644 index 0000000000000000000000000000000000000000..f14253032c94c7bdfe0d8cb7871164ec465dec63 --- /dev/null +++ b/doc/source/packages/packages.rst @@ -0,0 +1,68 @@ +Package preference +================== + +Initially we want our developers to following the +`coding guidelines for astropy-affiliated packages `_ +as much as possible. +A few important conventions and special cases should be outlined here. + +Basic preference +---------------- + +Several packages are favored over others if they can be used to solve the problem under study. +Developers should use them as much as possible. + + +Standard libraries + Python standard libraries have the highest priorities, e.g., ``os``, ``re``, etc. +``numpy``, ``scipy``, ``matplotlib`` + The ``BIG 3`` for Python scientific computing. +``astropy`` and its ``astropy``-affiliated packages + For example, ``astropy.io.fits`` is favored over ``pyfits``. + + +Parallel computing +------------------ + +The two packages are preferred for implementing `embarrassingly` parallel computing (without inter-communication). + +- ``multiprocessing``: https://docs.python.org/3/library/multiprocessing.html +- ``joblib``: https://joblib.readthedocs.io/en/latest/ + +.. literalinclude:: ./example_multiprocessing.py + :linenos: + :language: python + :caption: an example of using ``multiprocessing`` for parallel computing + +The output is + +.. code-block:: + + Total time cost: 5.095193147659302 sec! + +.. literalinclude:: ./example_joblib.py + :linenos: + :language: python + :caption: an example of using ``joblib`` for parallel computing + +The output is + +.. code-block:: + + [Parallel(n_jobs=5)]: Using backend LokyBackend with 5 concurrent workers. + [Parallel(n_jobs=5)]: Done 1 tasks | elapsed: 5.2s + [Parallel(n_jobs=5)]: Done 2 out of 5 | elapsed: 5.2s remaining: 7.8s + [Parallel(n_jobs=5)]: Done 3 out of 5 | elapsed: 5.2s remaining: 3.5s + [Parallel(n_jobs=5)]: Done 5 out of 5 | elapsed: 5.2s remaining: 0.0s + [Parallel(n_jobs=5)]: Done 5 out of 5 | elapsed: 5.2s finished + Total time cost: 5.1958301067352295 sec! + +.. tip:: + ``joblib`` is recommended for its highly concise syntax and verbose info -- do every thing within one statement. + ``n_jobs`` can be set to ``-1`` to use almost all CPUs, ``backend`` can be set to ``multiprocessing`` + to use the backend built by standard library ``multiprocessing``, or ``loky`` for alleged high robustness. + Visit https://joblib.readthedocs.io/en/latest/ for more info and usages of ``joblib``, + such as the ``batch_size`` and ``verbose`` parameters. + +For parallel computing with inter-communication or distributed computing, +we recommend developers to consider using ``mpi4py``: https://github.com/mpi4py/mpi4py. diff --git a/doc/source/packaging.rst b/doc/source/packaging.rst index 0f88f4a460fda3136e029b43b225bb7c7de86689..d5ca767d25dc6e680b02e2e6c6eca8388ea526f8 100644 --- a/doc/source/packaging.rst +++ b/doc/source/packaging.rst @@ -155,6 +155,14 @@ and in ``flip_image.py``: :language: python +Including C code +---------------- + +To include C code, developers should follow the guide provided by ``setuptools``: +https://setuptools.pypa.io/en/latest/userguide/ext_modules.html. +And also consider using https://docs.astropy.org/en/latest/development/ccython.html#building-c-or-cython-extensions. + + ``top_level_interface`` module ------------------------------ This is a special requirement for all of CSST DAS packages.