Commits (2)
...@@ -73,7 +73,7 @@ The function will return a successfull response as soon as the file content is s ...@@ -73,7 +73,7 @@ The function will return a successfull response as soon as the file content is s
A successfull response contains a task_id referring to the queued processing task. This can be used in [4. Query a L2 Processing Tasks State](#4-query-a-l2-processing-tasks-state) for querying a processing task's current state. A successfull response contains a task_id referring to the queued processing task. This can be used in [4. Query a L2 Processing Tasks State](#4-query-a-l2-processing-tasks-state) for querying a processing task's current state.
## Configuration ## Configuration
The helper will send HTTP requests to an external API. The CSST_BACKEND_API_URL env variable should be set accordingly. E.g. The helper will send HTTP requests to an external API. The external api url is set to working default values. It can be overwritten if needed via env variable, e.g.
CSST_BACKEND_API_URL=http://10.200.60.199:9010 CSST_BACKEND_API_URL=http://10.200.60.199:9010
## Function: `start_ingestion_task` ## Function: `start_ingestion_task`
...@@ -102,7 +102,8 @@ def start_ingestion_task(file_content: str, file_name: str) -> dict: ...@@ -102,7 +102,8 @@ def start_ingestion_task(file_content: str, file_name: str) -> dict:
Query for file info by metadata values. Query for file info by metadata values.
## Configuration ## Configuration
The helper will send HTTP requests to an external API. CSST_BACKEND_API_URL env variable should be set accordingly. The helper will send HTTP requests to an external API. The external api url is set to working default values. It can be overwritten if needed via env variable, e.g.
CSST_BACKEND_API_URL=http://10.200.60.199:9010
## Function: `query_metadata` ## Function: `query_metadata`
```python ```python
...@@ -183,7 +184,8 @@ filter = { ...@@ -183,7 +184,8 @@ filter = {
Query the processing state of a processing task given a L2 task id. Query the processing state of a processing task given a L2 task id.
## Configuration ## Configuration
The helper will send HTTP requests to an external API. CSST_BACKEND_API_URL env variable should be set accordingly. The helper will send HTTP requests to an external API. The external api url is set to working default values. It can be overwritten if needed via env variable, e.g.
CSST_BACKEND_API_URL=http://10.200.60.199:9010
## Function: `query_task_state` ## Function: `query_task_state`
```python ```python
...@@ -209,7 +211,8 @@ def query_task_state( ...@@ -209,7 +211,8 @@ def query_task_state(
Query a star catalog by column values given a ra, dec and radius preselection. Query a star catalog by column values given a ra, dec and radius preselection.
## Configuration ## Configuration
The helper will send HTTP requests to an external API. CSST_BACKEND_API_URL env variable should be set accordingly. The helper will send HTTP requests to an external API. The external api url is set to working default values. It can be overwritten if needed via env variable, e.g.
CSST_BACKEND_API_URL=http://10.200.60.199:9010
## Function: `query_star_catalog` ## Function: `query_star_catalog`
```python ```python
......
import os import os
import requests import requests
from typing import Dict, Any, List from typing import Dict, Any, List
from ..s3_config import load_backend_settings
def query_metadata( def query_metadata(
filter: Dict[str, Any], filter: Dict[str, Any],
...@@ -34,9 +35,9 @@ def query_metadata( ...@@ -34,9 +35,9 @@ def query_metadata(
if not filter: if not filter:
raise ValueError("Filter cannot be empty") raise ValueError("Filter cannot be empty")
api_url = os.getenv("CSST_BACKEND_API_URL") api_url = load_backend_settings()['backend_url']
if not api_url: if not api_url:
raise RuntimeError("CSST_BACKEND_API_URL environment variable is not set") raise RuntimeError("CSST backend api url is not set")
endpoint = f"{api_url}/datasync/metadata/query" endpoint = f"{api_url}/datasync/metadata/query"
......
import os import os
import requests import requests
from typing import Dict, Any, List from typing import Dict, Any, List
from ..s3_config import load_backend_settings
def query_star_catalog( def query_star_catalog(
catalog_name: str, catalog_name: str,
...@@ -33,9 +34,9 @@ def query_star_catalog( ...@@ -33,9 +34,9 @@ def query_star_catalog(
if not key: if not key:
raise ValueError("Key list cannot be empty") raise ValueError("Key list cannot be empty")
api_url = os.getenv("CSST_BACKEND_API_URL") api_url = load_backend_settings()['backend_url']
if not api_url: if not api_url:
raise RuntimeError("CSST_BACKEND_API_URL environment variable is not set") raise RuntimeError("CSST backend api url is not set")
endpoint = f"{api_url}/starcatalog/query" endpoint = f"{api_url}/starcatalog/query"
......
...@@ -4,6 +4,7 @@ import requests ...@@ -4,6 +4,7 @@ import requests
import time import time
from typing import Dict, Any from typing import Dict, Any
from csst_fs import s3_fs from csst_fs import s3_fs
from ..s3_config import load_backend_settings
def start_ingestion_task(file_content: bytes, file_name: str) -> Dict[str, str]: def start_ingestion_task(file_content: bytes, file_name: str) -> Dict[str, str]:
""" """
...@@ -26,9 +27,9 @@ def start_ingestion_task(file_content: bytes, file_name: str) -> Dict[str, str]: ...@@ -26,9 +27,9 @@ def start_ingestion_task(file_content: bytes, file_name: str) -> Dict[str, str]:
Raises: Raises:
RuntimeError: If the ingestion API or OSS upload fails after retries. RuntimeError: If the ingestion API or OSS upload fails after retries.
""" """
api_url = os.getenv("CSST_BACKEND_API_URL") api_url = load_backend_settings()['backend_url']
if not api_url: if not api_url:
raise RuntimeError("CSST_BACKEND_API_URL environment variable is not set") raise RuntimeError("CSST backend api url is not set")
# Step 1: Request an OSS upload path # Step 1: Request an OSS upload path
request_upload_endpoint = f"{api_url}/datasync/level2/upload" request_upload_endpoint = f"{api_url}/datasync/level2/upload"
...@@ -98,9 +99,9 @@ def query_task_state(task_id: str) -> Dict[str, Any]: ...@@ -98,9 +99,9 @@ def query_task_state(task_id: str) -> Dict[str, Any]:
if not task_id: if not task_id:
raise ValueError("task_id must be provided") raise ValueError("task_id must be provided")
api_url = os.getenv("CSST_BACKEND_API_URL") api_url = load_backend_settings()['backend_url']
if not api_url: if not api_url:
raise RuntimeError("CSST_BACKEND_API_URL environment variable is not set") raise RuntimeError("CSST backend api url is not set")
endpoint = f"{api_url}/datasync/level2/{task_id}" endpoint = f"{api_url}/datasync/level2/{task_id}"
......
...@@ -10,6 +10,10 @@ default_s3_settings = { ...@@ -10,6 +10,10 @@ default_s3_settings = {
'bucket': 'data-and-computing' 'bucket': 'data-and-computing'
} }
default_backend_settings = {
'backend_url': 'https://astro-workbench-bts.lab.zverse.space:32443/api/csst',
}
def load_from_env(): def load_from_env():
s3_options = { s3_options = {
'key': os.getenv('S3_KEY'), 'key': os.getenv('S3_KEY'),
...@@ -24,6 +28,12 @@ def load_settings_from_env(): ...@@ -24,6 +28,12 @@ def load_settings_from_env():
} }
return s3_settings return s3_settings
def load_backend_settings_from_env():
backend_settings = {
'backend_url': os.getenv('CSST_BACKEND_API_URL'),
}
return backend_settings
def load_s3_options(): def load_s3_options():
if 'S3_KEY' in os.environ: if 'S3_KEY' in os.environ:
return load_from_env() return load_from_env()
...@@ -33,3 +43,8 @@ def load_s3_settings(): ...@@ -33,3 +43,8 @@ def load_s3_settings():
if 'S3_BUCKET' in os.environ: if 'S3_BUCKET' in os.environ:
return load_settings_from_env() return load_settings_from_env()
return default_s3_settings return default_s3_settings
def load_backend_settings():
if 'CSST_BACKEND_API_URL' in os.environ:
return load_backend_settings_from_env()
return default_backend_settings