diff --git a/csst_common/data_manager.py b/csst_common/data_manager.py index c82cdb5b8f2fc88f8e645f3637b26a51bc5b4d36..65905712cdd4b13f69510fa901be14d6540fc56a 100644 --- a/csst_common/data_manager.py +++ b/csst_common/data_manager.py @@ -60,7 +60,7 @@ class CsstMsDataManager: Examples -------- - >>> dm_mbi = CsstMbiDataManager(...) + >>> dm_mbi = CsstMsDataManager(...) >>> # access L0 directory >>> dm_mbi.dir_l0 >>> # access L1 directory diff --git a/csst_common/file_recorder.py b/csst_common/file_recorder.py new file mode 100644 index 0000000000000000000000000000000000000000..7a486a783521a4f3c8c996603f7d0e9439bc3b5a --- /dev/null +++ b/csst_common/file_recorder.py @@ -0,0 +1,76 @@ +from collections import namedtuple +from astropy import table +import os + + +File = namedtuple( + "FileRecord", + ["filepath", "db", "comment", "existence"] +) + + +class FileRecorder(list): + """ + File Recorder, inherited from the built-in ``list``. + + This recorder is used to record files generated by functional modules. + In principle, a CSST pipeline module should return a status (CsstStatus) + and a file recorder (``FileRecorder``) after it finishes data processing. + + Methods + ------- + add_record(filepath: str = "", db: bool = False, comment: str = "") + Add a file record, each record is a ``collections.namedtuple``. + Users should provide a file path ``filepath``, whether this file should be + written to database ``db``, a comment ``comment``. + An existence boolean will be attached to check if the file exists. + to_table() + Convert to ``astropy.table.Table``. + pprint(*args, **kwargs) + Use ``astropy.table.Table.pprint`` to print file records in table format. + pprint_all(*args, **kwargs) + Use ``astropy.table.Table.pprint_all`` to print file records in table format. + + Examples + -------- + >>> fr = FileRecorder() + >>> for i in range(3): + >>> fr.add_record("test{:03d}.txt".format(i), db=True, comment="Test file {:d}".format(i)) + >>> fr.pprint_all() + + filepath db comment existence + ----------- ---- ----------- --------- + test000.txt True Test file 0 False + test001.txt True Test file 1 False + test002.txt True Test file 2 False + >>> fr.pprint_all() + + filepath db comment existence + ----------- ---- ----------- --------- + test000.txt True Test file 0 False + test001.txt True Test file 1 False + test002.txt True Test file 2 False + """ + + def __init__(self): + super(FileRecorder, self).__init__() + + def add_record(self, filepath: str = "", db: bool = False, comment: str = ""): + existence = os.path.exists(filepath) + assert isinstance(filepath, str) + assert isinstance(db, bool) + assert isinstance(comment, str) + super().append(File(filepath=filepath, db=db, comment=comment, existence=existence)) + + def to_table(self): + return table.Table([_._asdict() for _ in self]) + + def __repr__(self): + return self.to_table().__repr__() + + def pprint(self, *args, **kwargs): + return self.to_table().pprint(*args, **kwargs) + + def pprint_all(self, *args, **kwargs): + print("".format(len(self))) + return self.to_table().pprint_all(*args, **kwargs) diff --git a/csst_common/top_level_interface.py b/csst_common/top_level_interface.py index 94e60c778e138068eb9b2ab408c0327527fe9797..5e5c222ad55f194c51f8afe599a4e723f969c7e2 100644 --- a/csst_common/top_level_interface.py +++ b/csst_common/top_level_interface.py @@ -1,5 +1,6 @@ from .data_manager import CsstMsDataManager from .logger import get_logger from .status import CsstStatus +from .file_recorder import FileRecorder -__all__ = ["CsstMsDataManager", "get_logger"] +__all__ = ["CsstMsDataManager", "get_logger", "FileRecorder"] diff --git a/tests/test_file_recorder.py b/tests/test_file_recorder.py new file mode 100644 index 0000000000000000000000000000000000000000..9848d35aabe50146ed5d4d965dcbf0bb73d6c4c6 --- /dev/null +++ b/tests/test_file_recorder.py @@ -0,0 +1,22 @@ +""" +Identifier: KSC-SJ4-tests/test_data_manager.py +Name: test_data_manager.py +Description: data manager unit test +Author: Bo Zhang +Created: 2022-09-13 +Modified-History: + 2022-10-10, Bo Zhang, created + 2022-10-10, Bo Zhang, added test for FileRecorder +""" +import unittest +from csst_common.file_recorder import FileRecorder, File + + +# deprecated in some days +class TestCsstMbiDataManager(unittest.TestCase): + def setUp(self) -> None: + self.fr = FileRecorder() + + def test_add_record(self): + self.fr.add_record(filepath="test1", db=False, comment="the first test") + self.assertTrue(self.fr[0] == File("test1", db=False, comment="the first test", existence=False))