Skip to content
GitLab
Commit 9345796e authored by Weidenthaler Matthias's avatar Weidenthaler Matthias
Browse files

Merge branch 'search-and-commit-api'

parents 476195e4 120148f4
Pipeline #11293 passed with stage
Hide whitespace changes
Inline Side-by-side
README.md
View file @ 9345796e
......@@ -3,8 +3,8 @@ This repository provides the following functionalities:
2. [Commit For File Processing](#2-commit-for-file-processing)
3. [Query a List Of L1/L2 Fits-Files By Metadata Values](#3-query-a-list-of-l1l2-fits-files-by-metadata-values)
4. [Query a L2 Processing Tasks State](#4-query-a-l2-processing-tasks-state)
5. [Query a Star Catalog](#5-query-a-star-catalog)
6. [Query a Star Catalog Corresponding to Metadata Entries](#6-query-a-star-catalog-corresponding-to-metadata-entries)
5. [Query a Catalog](#5-query-a-catalog)
6. [Query a Catalog Corresponding to Metadata Entries](#6-query-a-catalog-corresponding-to-metadata-entries)
7. [Trigger a Pipeline Run](#7-trigger-a-pipeline-run)
8. [Query a Pipeline Run State](#8-query-a-pipeline-run-state)
......@@ -205,21 +205,21 @@ def query_task_state(
"""
```
# 5. Query a Star Catalog
Query a star catalog by column values given a ra, dec and radius preselection.
# 5. Query a Catalog
Query a catalog by column values given a ra, dec and radius preselection.
## Function: `query_star_catalog`
## Function: `query_catalog`
```python
def query_star_catalog(
def query_catalog(
catalog_name: str,
filter: Dict[str, Any],
key: List[str],
) -> List[Dict[str, Any]]:
"""
Query a star catalog by column values given a ra, dec and radius preselection.
Query a catalog by column values given a ra, dec and radius preselection.
Args:
catalog_name: Name of the star catalog (e.g. csst-msc-l1-mbi-catmix)
catalog_name: Name of the catalog (e.g. csst-msc-l1-mbi-catmix)
filter: The filter dict described below.
The following keys MUST be set:
{
......@@ -228,11 +228,11 @@ def query_star_catalog(
"radius": 0.2,
}
Ra, dec values pinpoint a location, 'radius' defines a radius in [deg] around this point.
Only star catalog objects withing this area are considered for subsequent filtering.
Only catalog objects withing this area are considered for subsequent filtering.
Setting ranges with (lt, gt, lte, gte) for ra, dec values is not supported.
key: A list of string values, corresponding to the colum names that should be present in the return value.
Returns:
A List[Dict] of matching star catalog objects, containing key-value pairs for the keys set as 'key' parameter.
A List[Dict] of matching catalog objects, containing key-value pairs for the keys set as 'key' parameter.
E.g. with key = ["x", "bulge_flux", "ab"]
then returns:
[
......@@ -245,18 +245,18 @@ def query_star_catalog(
"""
```
# 6. Query a Star Catalog Corresponding to Metadata Entries
First queries the metadata catalog, based on that subsequently queries the star catalog.
# 6. Query a Catalog Corresponding to Metadata Entries
First queries the metadata catalog, based on that subsequently queries the catalog.
## Function `query_star_catalog_with_metadata`
## Function `query_catalog_with_metadata`
```python
def query_star_catalog_with_metadata(
def query_catalog_with_metadata(
metadata: Dict[str, Any],
star_catalog: Dict[str, Any],
catalog: Dict[str, Any],
) -> List[Dict[str, Any]]:
"""
Queries the metadata catalog according to the provided filter criteria and HDU value.
Subsequently queries the star catalog entries corresponding to the metadata results and
Subsequently queries the catalog entries corresponding to the metadata results and
the given additional filters.
Returns the catalog columns specified in the 'key' list.
......@@ -265,8 +265,8 @@ def query_star_catalog_with_metadata(
filter: filter dict described below.
hdu: The hdu the filter & key arguments refer to. Default is 0. E.g. 0, 1.
},
star_catalog: {
catalog_name: Name of the star catalog (e.g. csst-msc-l1-mbi-catmix)
catalog: {
catalog_name: Name of the catalog (e.g. csst-msc-l1-mbi-catmix)
filter: filter dict described below.
The following keys MUST be set:
{
......@@ -280,8 +280,8 @@ def query_star_catalog_with_metadata(
Example:
from csst_fs import *
query_star_catalog_with_metadata(
star_catalog={
query_catalog_with_metadata(
catalog={
"catalogName": "csst-msc-l1-mbi-catmix",
"key": ["data_uuid", "obsid", "ra"],
"filter": {"ra": 130.97, "dec": -20.5, "radius": 0.09, "x": {"lt": 30}},
......@@ -293,7 +293,7 @@ def query_star_catalog_with_metadata(
)
Returns:
A List[Dict] of matching star catalog objects.
A List[Dict] of matching catalog objects.
"""
```
......
csst_fs/__init__.py
View file @ 9345796e
......@@ -5,6 +5,6 @@ from .table import fsspec_table
from . import s3_fs
from . import fs
from .catalog.metadata import query_metadata
from .catalog.star import query_star_catalog, query_star_catalog_with_metadata
from .catalog.star import query_catalog, query_catalog_with_metadata
from .ingestion.level2 import start_ingestion_task, query_task_state
from .pipeline.pipeline import run_pipeline, PipelineBatch, query_run_state
\ No newline at end of file
csst_fs/catalog/star.py
View file @ 9345796e
......@@ -4,7 +4,7 @@ from typing import Dict, Any, List
from pydantic import BaseModel, Field, validator
from ..s3_config import load_backend_settings
class StarCatalogQuery(BaseModel):
class CatalogQuery(BaseModel):
catalogName: str
key: List[str]
filter: Dict[str, Any]
......@@ -21,16 +21,16 @@ class MetadataQuery(BaseModel):
class CombinedCatalogMetadataRequest(BaseModel):
starCatalog: StarCatalogQuery
starCatalog: CatalogQuery
metadata: MetadataQuery
def query_star_catalog_with_metadata(
def query_catalog_with_metadata(
metadata: Dict[str, Any],
star_catalog: Dict[str, Any],
catalog: Dict[str, Any],
) -> List[Dict[str, Any]]:
"""
Queries the metadata catalog according to the provided filter criteria and HDU value.
Subsequently queries the star catalog entries corresponding to the metadata results and
Subsequently queries the catalog entries corresponding to the metadata results and
the given additional filters.
Returns the catalog columns specified in the 'key' list.
......@@ -39,8 +39,8 @@ def query_star_catalog_with_metadata(
filter: filter dict described in the README filter syntax section.
hdu: The hdu the filter & key arguments refer to. Default is 0. E.g. 0, 1.
},
star_catalog: {
catalog_name: Name of the star catalog (e.g. csst-msc-l1-mbi-catmix)
catalog: {
catalog_name: Name of the catalog (e.g. csst-msc-l1-mbi-catmix)
filter: filter dict described in the README filter syntax section.
The following keys MUST be set:
{
......@@ -54,8 +54,8 @@ def query_star_catalog_with_metadata(
Example:
from csst_fs import *
query_starcatalog_with_metadata(
... star_catalog={
query_catalog_with_metadata(
... catalog={
... "catalogName": "csst-msc-l1-mbi-catmix",
... "key": ["data_uuid", "obsid", "ra"],
... "filter": {"ra": 130.97, "dec": -20.5, "radius": 0.09, "x": {"lt": 30}},
......@@ -67,7 +67,7 @@ def query_star_catalog_with_metadata(
... )
Returns:
A List[Dict] of matching star catalog objects.
A List[Dict] of matching catalog objects.
"""
api_url = load_backend_settings()['backend_url']
......@@ -76,11 +76,11 @@ def query_star_catalog_with_metadata(
endpoint = f"{api_url}/starcatalog/query/metadata"
star_catalog_model = StarCatalogQuery(**star_catalog)
catalog_model = CatalogQuery(**catalog)
metadata_model = MetadataQuery(**metadata)
payload = CombinedCatalogMetadataRequest(
starCatalog=star_catalog_model,
starCatalog=catalog_model,
metadata=metadata_model,
).dict()
......@@ -94,16 +94,16 @@ def query_star_catalog_with_metadata(
return data["result"]
def query_star_catalog(
def query_catalog(
catalog_name: str,
filter: Dict[str, Any],
key: List[str],
) -> List[Dict[str, Any]]:
"""
Query a star catalog by column values given a ra, dec and radius preselection.
Query a catalog by column values given a ra, dec and radius preselection.
Args:
catalog_name: Name of the star catalog (e.g. csst-msc-l1-mbi-catmix)
catalog_name: Name of the catalog (e.g. csst-msc-l1-mbi-catmix)
filter: The filter dict described below.
The following keys MUST be set:
{
......@@ -115,7 +115,7 @@ def query_star_catalog(
key: A list of string values, corresponding to the column names that should be present in the return value.
Returns:
A List[Dict] of matching star catalog objects.
A List[Dict] of matching catalog objects.
"""
for required_key in ("ra", "dec", "radius"):
......@@ -141,7 +141,7 @@ def query_star_catalog(
response = requests.post(endpoint, json=payload, timeout=30)
response.raise_for_status()
except requests.RequestException as e:
raise RuntimeError(f"Star catalog query failed: {e}")
raise RuntimeError(f"Catalog query failed: {e}")
data = response.json()
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment