added FileRecorder to documentation (5cbab9d2) · Commits · csst-pipeline / csst_proto

doc/source/csst_common/csst_common.rst

+27 −1

Original line number	Original line	Diff line number	Diff line
	@@ -97,6 +97,32 @@ raw 0 ``dm.l0_log(detector=detector)``
	========================== ===== ========================================================== ==================		========================== ===== ========================================================== ==================


			``csst_common.file_recorder.FileRecorder``
			------------------------------------------

			Get an empty ``FileRecorder``.
			This is initially a ``list``-like object.
			Use ``FileRecorder.add_record()`` to add file records.

			.. code-block::
			:linenos:

			>>> from csst_common.file_recorder import FileRecorder
			>>> fr = FileRecorder()
			>>> for i in range(3):
			>>> fr.add_record(filepath="test{:03d}.txt".format(i),
			>>> db=True,
			>>> comment="Test file {:d}".format(i))
			>>> fr.pprint_all()

			<FileRecorder length=3>
			filepath db comment existence
			----------- ---- ----------- ---------
			test000.txt True Test file 0 False
			test001.txt True Test file 1 False
			test002.txt True Test file 2 False


	``csst_common.logger.get_logger()``		``csst_common.logger.get_logger()``
	-----------------------------------		-----------------------------------

	@@ -147,7 +173,7 @@ Source code

	.. literalinclude:: example_interface.py		.. literalinclude:: example_interface.py
	:caption: ``example_interface.py``		:caption: ``example_interface.py``
	:emphasize-lines: 36-40,81-82,85-86,89-90,94-97,143-144,147,150-151,154-155		:emphasize-lines: 7-10,36-40,84,86-87,90-91,94-95,99,102,148,152-153,156-157,160-161,164-165
	:linenos:		:linenos:
	:language: python		:language: python

doc/source/csst_common/example_interface.py

+42 −32

Original line number	Original line	Diff line number	Diff line
	@@ -5,13 +5,14 @@ from typing import Union
	import numpy as np		import numpy as np
	from astropy.io import fits		from astropy.io import fits
	from csst_common.data_manager import CsstMsDataManager		from csst_common.data_manager import CsstMsDataManager
			from csst_common.file_recorder import FileRecorder
	from csst_common.logger import get_logger		from csst_common.logger import get_logger
	from csst_common.status import CsstStatus		from csst_common.status import CsstStatus


	def read_image(filename_input: str) -> np.ndarray:		def read_image(filepath_input: str) -> np.ndarray:
	""" Read image. """		""" Read image. """
	return fits.getdata(filename_input)		return fits.getdata(filepath_input)


	def process_data(data: np.ndarray) -> np.ndarray:		def process_data(data: np.ndarray) -> np.ndarray:
	@@ -28,16 +29,15 @@ def check_results(dm: CsstMsDataManager, logger: logging.Logger) -> bool:
	if all(existence):		if all(existence):
	return True		return True
	else:		else:
	logger.warning("Not all processed files are generated!")
	return False		return False


	# process a single image		# process a single image
	def process_single_image(		def process_single_image(
	filename_input: str,		filepath_input: str,
	filename_output: str,		filepath_output: str,
	logger: Union[None, logging.Logger] = None		logger: Union[None, logging.Logger] = None
	) -> CsstStatus:		) -> tuple[CsstStatus, FileRecorder]:
	"""		"""
	Flip a single image.		Flip a single image.

	@@ -45,23 +45,23 @@ def process_single_image(

	Parameters		Parameters
	----------		----------
	filename_input : str		filepath_input : str
	The input filename.		The input filepath.
	filename_output : str		filepath_output : str
	The output filename.		The output filepath.
	logger : logging.Logger		logger : logging.Logger
	The logger.		The logger.

	Returns		Returns
	-------		-------
	CsstStatus		tuple[CsstStatus, FileRecorder]
	The final status.		The final status.

	Examples		Examples
	--------		--------
	>>> process_single_image(		>>> process_single_image(
	>>> filename_input="input_image.fits",		>>> filepath_input="input_image.fits",
	>>> filename_output="output_image.fits",		>>> filepath_output="output_image.fits",
	>>> logger=None		>>> logger=None
	>>> )		>>> )
	"""		"""
	@@ -69,32 +69,37 @@ def process_single_image(
	if logger is None:		if logger is None:
	logger = get_logger()		logger = get_logger()

			# get an empty file recorder
			fr = FileRecorder()

	# process data		# process data
	try:		try:
	# this will NOT be written into the log file		# this will NOT be written into the log file
	logger.debug("Reading the image {}".format(filename_input))		logger.debug("Reading the image {}".format(filepath_input))
	# start processing		# start processing
	data = read_image(filename_input)		data = read_image(filepath_input)
	data_processed = process_data(data)		data_processed = process_data(data)
	np.save(filename_output, data_processed)		np.save(filepath_output, data_processed)
			# record file!
			fr.add_record(filepath=filepath_output, db=True, comment="the processed image")
	# this will be written into the log file		# this will be written into the log file
	logger.info("Processed image saved to {}".format(filename_output))		logger.info("Processed image saved to {}".format(filepath_output))
	return CsstStatus.PERFECT		return CsstStatus.PERFECT, fr
	except DeprecationWarning:		except DeprecationWarning:
	# this will be written into the log file		# this will be written into the log file
	logger.warning("Suffered DeprecationWarning!")		logger.warning("Suffered DeprecationWarning!")
	return CsstStatus.WARNING		return CsstStatus.WARNING, fr
	except IOError:		except IOError:
	# this will be written into the log file		# this will be written into the log file
	logger.error("Suffered IOError!")		logger.error("Suffered IOError!")
	return CsstStatus.ERROR		return CsstStatus.ERROR, fr


	# process an exposure (MBI or SLS)		# process an exposure (MBI or SLS)
	def process_multiple_images(		def process_multiple_images(
	dm: CsstMsDataManager,		dm: CsstMsDataManager,
	logger: Union[None, logging.Logger] = None		logger: Union[None, logging.Logger] = None
	) -> CsstStatus:		) -> tuple[CsstStatus, FileRecorder]:
	"""		"""
	Flip all images.		Flip all images.

	@@ -109,7 +114,7 @@ def process_multiple_images(

	Returns		Returns
	-------		-------
	CsstStatus		tuple[CsstStatus, FileRecorder]
	The final status.		The final status.

	Examples		Examples
	@@ -124,6 +129,9 @@ def process_multiple_images(
	if logger is None:		if logger is None:
	logger = get_logger()		logger = get_logger()

			# get an empty file recorder
			fr = FileRecorder()

	# process data		# process data
	try:		try:
	# dm.target_detectors is a list of detector number that should be processed		# dm.target_detectors is a list of detector number that should be processed
	@@ -131,25 +139,27 @@ def process_multiple_images(
	for detector in dm.target_detectors:		for detector in dm.target_detectors:
	# this will NOT be written into the log file		# this will NOT be written into the log file
	logger.debug("Processing for detector {}".format(detector))		logger.debug("Processing for detector {}".format(detector))
	data = read_image(dm.l0_detector(detector=detector))		filepath_input = dm.l0_detector(detector=detector)
			filepath_output = dm.l1_detector(detector=detector, post="L1_processed.fits")
			data = read_image(filepath_input)
	data_processed = process_data(data)		data_processed = process_data(data)
	np.save(		np.save(filepath_output, data_processed)
	dm.l1_detector(detector=detector, post="L1_processed.fits"),		# record file!
	data_processed		fr.add_record(filepath=filepath_output, db=True, comment="processed file for Detector {}".format(detector))
	)
	# check results		# check results
	if check_results(dm=dm, logger=logger):		if check_results(dm=dm, logger=logger):
	# this will be written into the log file		# this will be written into the log file
	logger.info("Processed all images")		logger.info("All processed files are generated!")
	return CsstStatus.PERFECT		return CsstStatus.PERFECT, fr
	else:		else:
	# not all images are properly processed		# not all images are properly processed
	return CsstStatus.ERROR		logger.warning("Not all processed files are generated!")
			return CsstStatus.ERROR, fr
	except DeprecationWarning:		except DeprecationWarning:
	# this will be written into the log file		# this will be written into the log file
	logger.warning("Suffered DeprecationWarning!")		logger.warning("Suffered DeprecationWarning!")
	return CsstStatus.WARNING		return CsstStatus.WARNING, fr
	except IOError:		except IOError:
	# this will be written into the log file		# this will be written into the log file
	logger.error("Suffered IOError!")		logger.error("Suffered IOError!")
	return CsstStatus.ERROR		return CsstStatus.ERROR, fr