Add lazy logging to configurable logging directory

README.md

@@ -8,6 +8,24 @@ This repository provides the following functionalities:
 7. [Trigger a Pipeline Run](#7-trigger-a-pipeline-run)
 8. [Query a Pipeline Run State](#8-query-a-pipeline-run-state)
 
+## Logging Configuration
+By default, log files are written to the current working directory. You can configure a custom logging directory to centralize all csst_fs related log file outputs:
+
+```python
+import csst_fs
+
+# Configure logging directory (must be an absolute path)
+csst_fs.configure_logging('/var/log/csst_fs')
+
+# All subsequent log files will be written to the configured directory
+# This affects all loggers in the csst_fs package
+```
+
+**Important Notes:**
+- The logging directory must be an absolute path and will be created automatically if it doesn't exist.
+- Log files are created **lazily** - they are only created when the first log message is written, not during import.
+- Call `configure_logging()` early in your application to ensure all subsequent logs go to the desired directory.
+
 # 1. Read or Download a File from S3 storage
 Supported are two distinct ways of reading from s3 storage.
 1) [Download to a local file](#从s3下载到本地)

csst_fs/__init__.py

@@ -7,4 +7,5 @@ from . import fs
 from .catalog.metadata import query_metadata
 from .catalog.star import query_catalog, query_catalog_with_metadata
 from .ingestion.level2 import start_ingestion_task, query_task_state
-from .pipeline.pipeline import run_pipeline, PipelineBatch, query_run_state
\ No newline at end of file
+from .pipeline.pipeline import run_pipeline, PipelineBatch, query_run_state
+from .log_config import configure_logging, get_log_directory, get_log_file_path, LazyFileHandler
\ No newline at end of file

csst_fs/ingestion/level2.py

@@ -5,12 +5,14 @@ import json
 import logging
 from csst_fs import s3_fs
 from ..s3_config import load_backend_settings
+from .. import log_config
 
 logger = logging.getLogger(__name__)
 if not logger.handlers:
     logger.setLevel(logging.DEBUG)
-    # File handler for info, warnings and errors
-    fh = logging.FileHandler('csst_fs_ingestion.log')
+    # File handler for info, warnings and errors (lazy - file created on first log)
+    log_file_path = log_config.get_log_file_path('csst_fs_ingestion.log')
+    fh = log_config.LazyFileHandler(log_file_path)
     fh.setLevel(logging.DEBUG)
     # Console handler for warnings and errors
     ch = logging.StreamHandler()

csst_fs/log_config.py

+import logging
+import os
+from pathlib import Path
+from typing import Optional
+
+_log_directory: Optional[Path] = None
+
+
+class LazyFileHandler(logging.FileHandler):
+    """
+    A FileHandler that defers file creation until the first log record is emitted.
+    This prevents creating log files in the current directory if no logging occurs.
+    """
+    def __init__(self, filename, mode='a', encoding=None, delay=True):
+        """
+        Initialize the handler with delay=True to defer file opening.
+        """
+        # Always use delay=True to defer file creation
+        super().__init__(filename, mode, encoding, delay=True)
+
+    def emit(self, record):
+        """
+        Emit a record, opening the file if needed.
+        """
+        # The parent class will open the file on first emit if delay=True
+        super().emit(record)
+
+def configure_logging(path: str) -> None:
+    """
+    Configure the logging directory for all csst_fs loggers.
+
+    This function sets the absolute directory where log files will be written.
+    All loggers created after calling this function will write their file outputs
+    to files within this directory.
+
+    Args:
+        path: Absolute path to the logging directory. The directory will be
+              created if it doesn't exist.
+
+    Raises:
+        ValueError: If the path is not absolute.
+        OSError: If the directory cannot be created.
+
+    Example:
+        >>> import csst_fs
+        >>> csst_fs.configure_logging('/var/log/csst_fs')
+    """
+    global _log_directory
+
+    path_obj = Path(path)
+
+    if not path_obj.is_absolute():
+        raise ValueError(f"Logging path must be absolute, got: {path}")
+
+    # Create directory if it doesn't exist
+    path_obj.mkdir(parents=True, exist_ok=True)
+
+    _log_directory = path_obj
+
+    # Update all existing handlers
+    _update_existing_handlers()
+
+def get_log_directory() -> Optional[Path]:
+    """
+    Get the currently configured logging directory.
+
+    Returns:
+        Path object of the logging directory, or None if not configured.
+    """
+    return _log_directory
+
+def get_log_file_path(filename: str) -> str:
+    """
+    Get the full path for a log file.
+
+    Args:
+        filename: Name of the log file.
+
+    Returns:
+        Absolute path to the log file. If a logging directory is configured,
+        returns path within that directory. Otherwise, returns the filename
+        (which will create the log in the current working directory).
+    """
+    if _log_directory is not None:
+        return str(_log_directory / filename)
+    return filename
+
+def _update_existing_handlers() -> None:
+    """
+    Update all existing FileHandlers in csst_fs loggers to use the new directory.
+    Only affects loggers within the csst_fs package.
+    """
+    if _log_directory is None:
+        return
+
+    # Iterate through all loggers, but only update csst_fs loggers
+    for logger_name in logging.Logger.manager.loggerDict:
+        # Only process loggers that belong to the csst_fs package
+        if not logger_name.startswith('csst_fs'):
+            continue
+
+        logger = logging.getLogger(logger_name)
+        if not isinstance(logger, logging.Logger):
+            continue
+
+        for handler in logger.handlers[:]:
+            if isinstance(handler, logging.FileHandler):
+                # Get the base filename
+                old_path = Path(handler.baseFilename)
+                new_path = _log_directory / old_path.name
+
+                # Create a new lazy handler with the same settings
+                new_handler = LazyFileHandler(str(new_path))
+                new_handler.setLevel(handler.level)
+                new_handler.setFormatter(handler.formatter)
+
+                # Replace the old handler
+                logger.removeHandler(handler)
+                handler.close()
+                logger.addHandler(new_handler)