Added L2 commit API

csst_fs/__init__.py

@@ -3,4 +3,5 @@ from .fits import fsspec_header
 from .fits import fsspec_HDUList
 from .table import fsspec_table
 from . import s3_fs
-from . import fs
+from . import fs
\ No newline at end of file
+from .commit import level2
\ No newline at end of file

csst_fs/commit/level2.py

+import os
+import base64
+import requests
+import time
+
+def submit_file_for_ingestion(file_content: bytes, file_name: str) -> dict:
+    """
+    Submit a file's content and file name to the ingestion API with retry logic.
+
+    Args:
+        file_content (bytes): The file's content as bytes
+        file_name (str): The file name for storing the file after ingestion.
+
+    Returns:
+        dict: A dict containing a task_id referring to the queued processing task's id.
+        Example:
+        {
+            "task_id": "5",
+        }
+
+    Raises:
+        RuntimeError: If the ingestion API returns an error after retries or request fails.
+    """
+
+    api_url = os.getenv("INGESTION_API_URL")
+    if not api_url:
+        raise RuntimeError("INGESTION_API_URL environment variable is not set")
+
+    endpoint = f"{api_url}/datasync/level2"
+    encoded_content = base64.b64encode(file_content).decode("utf-8")
+    payload = {
+        "fileContent": encoded_content,
+        "fileName": file_name
+    }
+
+    max_retries = 3
+    backoff_factor = 2
+
+    for attempt in range(1, max_retries + 1):
+        try:
+            response = requests.post(endpoint, json=payload, timeout=30)
+            response.raise_for_status()
+            data = response.json()
+
+            if data.get("success") and "result" in data and "task_id" in data["result"]:
+                return {"task_id": data["result"]["task_id"]}
+            else:
+                raise RuntimeError(f"Ingestion API returned an error: {data}")
+
+        except (requests.RequestException, RuntimeError) as e:
+            print(e)
+            if attempt == max_retries:
+                raise RuntimeError(f"Failed after {max_retries} attempts: {e}")
+            else:
+                wait_time = backoff_factor ** (attempt - 1)
+                print(f"Attempt {attempt} failed, retrying in {wait_time}s...")
+                time.sleep(wait_time)
\ No newline at end of file

setup.py

@@ -7,7 +7,8 @@ setup(
     install_requires=[
         'astropy>=5.3',
         'fsspec>=2024.5.0',
-        's3fs>=2024.5.0'
+        's3fs>=2024.5.0',
+        'requests'
     ],
     python_requires='>=3.9',
     description='csst pipeline handle file in local file system and remote s3  file system',