levelco and gaiadr3

csst_dfs_commons/models/common.py

 import copy
 from dataclasses import dataclass, field
+import json
 
 def from_proto_model_list(clazz, records):
     return [clazz().from_proto_model(r) for r in records]
@@ -11,10 +12,22 @@ def to_proto_model_list(protoObjClazz, records):
     return [r.to_proto_model(protoObjClazz) for r in records]
 
 def default_field(obj):
-    return field(default_factory=lambda: copy.copy(obj))    
+    return field(default_factory=lambda: copy.copy(obj))
+
+class Dict(dict):
+    __setattr__ = dict.__setitem__
+    __getattr__ = dict.__getitem__
+ 
+def dict_to_object(dictObj):
+    if not isinstance(dictObj, dict):
+        return dictObj
+    inst=Dict()
+    for k,v in dictObj.items():
+        inst[k] = dict_to_object(v)
+    return inst
+
 @dataclass
 class BaseModel:
-
     def from_dict(self, data = {}):
         for k in self.__dataclass_fields__.keys():
             self.__setattr__(k, data.get(k, None))
@@ -24,7 +37,10 @@ class BaseModel:
         if record is None:
             return None
         for k in self.__dataclass_fields__.keys():
-            self.__setattr__(k, record.__getattribute__(k))
+            if k == 'header':
+                self.__setattr__(k, json.loads(record.__getattribute__(k)))
+            else:
+                self.__setattr__(k, record.__getattribute__(k))
         return self
     
     def to_proto_model(self, protoObjClazz):
@@ -35,19 +51,24 @@ class BaseModel:
                 if isinstance(v, list) or isinstance(v, tuple):
                     obj.__getattribute__(k).extend(v)
                 elif isinstance(v, dict):
-                    obj.__getattribute__(k).update(v)                    
+                    if k == 'header':
+                        obj.__setattr__(k, json.dumps(v))
+                    else:
+                        obj.__getattribute__(k).update(v)                    
                 else:
                     obj.__setattr__(k, v)
         return obj
 
 @dataclass
 class Gaia3Record(BaseModel):
-    SourceId: int  = -1
-    RandomIndex: int  = -1
+    SolutionId: int = -1
+    Designation: str = ''
+    SourceId: int = -1
+    RandomIndex: int = -1
     RefEpoch: float = -1
     Ra: float = -1
     RaError: float = -1
-    Dec : float = -1
+    Dec: float = -1
     DecError: float = -1
     Parallax: float = -1
     ParallaxError: float = -1
@@ -57,7 +78,7 @@ class Gaia3Record(BaseModel):
     PmraError: float = -1
     Pmdec: float = -1
     PmdecError: float = -1
-    RaDecCorr : float = -1
+    RaDecCorr: float = -1
     RaParallaxCorr: float = -1
     RaPmraCorr: float = -1
     RaPmdecCorr: float = -1
@@ -76,18 +97,18 @@ class Gaia3Record(BaseModel):
     AstrometricExcessNoise: float = -1
     AstrometricExcessNoiseSig: float = -1
     AstrometricParamsSolved: int = -1
-    AstrometricPrimaryFlag: int = -1
+    AstrometricPrimaryFlag: int = 0
     NuEffUsedInAstrometry: float = -1
     Pseudocolour: float = -1
     PseudocolourError: float = -1
     RaPseudocolourCorr: float = -1
     DecPseudocolourCorr: float = -1
     ParallaxPseudocolourCorr: float = -1
-    PmraPseudocolourCorr : float = -1
+    PmraPseudocolourCorr: float = -1
     PmdecPseudocolourCorr: float = -1
     AstrometricMatchedTransits: int = -1
     VisibilityPeriodsUsed: int = -1
-    AstrometricSigma5dMax : float = -1
+    AstrometricSigma5dMax: float = -1
     MatchedTransits: int = -1
     NewMatchedTransits: int = -1
     MatchedTransitsRemoved: int = -1
@@ -104,7 +125,7 @@ class Gaia3Record(BaseModel):
     ScanDirectionMeanK2: float = -1
     ScanDirectionMeanK3: float = -1
     ScanDirectionMeanK4: float = -1
-    DuplicatedSource: int = -1
+    DuplicatedSource: int = 0
     PhotGNObs: int = -1
     PhotGMeanFlux: float = -1
     PhotGMeanFluxError: float = -1
@@ -120,29 +141,83 @@ class Gaia3Record(BaseModel):
     PhotRpMeanFluxError: float = -1
     PhotRpMeanFluxOverError: float = -1
     PhotRpMeanMag: float = -1
+    PhotBpRpExcessFactor: float = -1
     PhotBpNContaminatedTransits: float = -1
     PhotBpNBlendedTransits: float = -1
     PhotRpNContaminatedTransits: float = -1
     PhotRpNBlendedTransits: float = -1
-    PhotProcMode : float = -1
-    PhotBpRpExcessFactor: float = -1
+    PhotProcMode: float = -1
     BpRp: float = -1
-    BpG : float = -1
-    GRp : float = -1
-    Dr2RadialVelocity: float = -1
-    Dr2RadialVelocityError: float = -1
-    Dr2RvNbTransits: int = -1
-    Dr2RvTemplateTeff: float = -1
-    Dr2RvTemplateLogg: float = -1
-    Dr2RvTemplateFeH : float = -1
+    BpG: float = -1
+    GRp: float = -1
+    RadialVelocity: float = -1
+    RadialVelocityError: float = -1
+    RvMethodUsed: float = -1
+    RvNbTransits: float = -1
+    RvNbDeblendedTransits: float = -1
+    RvVisibilityPeriodsUsed: float = -1
+    RvExpectedSigToNoise: float = -1
+    RvRenormalisedGof: float = -1
+    RvChisqPvalue: float = -1
+    RvTimeDuration: float = -1
+    RvAmplitudeRobust: float = -1
+    RvTemplateTeff: float = -1
+    RvTemplateLogg: float = -1
+    RvTemplateFeH: float = -1
+    RvAtmParamOrigin: float = -1
+    Vbroad: float = -1
+    VbroadError: float = -1
+    VbroadNbTransits: float = -1
+    GrvsMag: float = -1
+    GrvsMagError: float = -1
+    GrvsMagNbTransits: float = -1
+    RvsSpecSigToNoise: float = -1
+    PhotVariableFlag: str = ''
     L: float = -1
     B: float = -1
     EclLon: float = -1
     EclLat: float = -1
+    InQsoCandidates: int = 0
+    InGalaxyCandidates: int = 0
+    NonSingleStar: int = -1
+    HasXpContinuous: int = 0
+    HasXpSampled: int = 0
+    HasRvs: int = 0
+    HasEpochPhotometry: int = 0
+    HasEpochRv: int = 0
+    HasMcmcGspphot: int = 0
+    HasMcmcMsc: int = 0
+    InAndromedaSurvey: int = 0
+    ClassprobDscCombmodQuasar: float = -1
+    ClassprobDscCombmodGalaxy: float = -1
+    ClassprobDscCombmodStar: float = -1
+    TeffGspphot: float = -1
+    TeffGspphotLower: float = -1
+    TeffGspphotUpper: float = -1
+    LoggGspphot: float = -1
+    LoggGspphotLower: float = -1
+    LoggGspphotUpper: float = -1
+    MhGspphot: float = -1
+    MhGspphotLower: float = -1
+    MhGspphotUpper: float = -1
+    DistanceGspphot: float = -1
+    DistanceGspphotLower: float = -1
+    DistanceGspphotUpper: float = -1
+    AzeroGspphot: float = -1
+    AzeroGspphotLower: float = -1
+    AzeroGspphotUpper: float = -1
+    AgGspphot: float = -1
+    AgGspphotLower: float = -1
+    AgGspphotUpper: float = -1
+    EbpminrpGspphot: float = -1
+    EbpminrpGspphotLower: float = -1
+    EbpminrpGspphotUpper: float = -1
+    LibnameGspphot: str = ''
     NS8HIdx: int = -1
     NS16HIdx: int = -1
     NS32HIdx: int = -1
     NS64HIdx: int = -1
-
+    FileIdx: int = -1
+    
     def __init__(self):
         self.AstrometricNObsAc = 0

csst_dfs_commons/models/cpic.py

@@ -30,6 +30,7 @@ class Level0Record(BaseModel):
     prc_status: int = 0
     prc_time: str=""
     create_time: str=""
+    header: Dict[str,object] = default_field({})
 
 @dataclasses.dataclass
 class CalMergeRecord(BaseModel):

csst_dfs_commons/models/ifs.py

@@ -13,14 +13,6 @@ class Level0PrcRecord(BaseModel):
     prc_time: str=""
     result_file_path: str=""
 
-@dataclasses.dataclass
-class Level0HeaderRecord(BaseModel):
-    id: int = 0
-    object_name: str = ""
-    ra: float = 0
-    dec: float = 0
-    version: str = ""
-
 @dataclasses.dataclass
 class Level0Record(BaseModel):
     id: int = 0
@@ -38,6 +30,7 @@ class Level0Record(BaseModel):
     prc_status: int = 0
     prc_time: str=""
     create_time: str=""
+    header: Dict[str,object] = default_field({})
 
 @dataclasses.dataclass
 class CalMergeRecord(BaseModel):

csst_dfs_commons/models/mci.py

@@ -13,14 +13,6 @@ class Level0PrcRecord(BaseModel):
     prc_time: str=""
     result_file_path: str=""
 
-@dataclasses.dataclass
-class Level0HeaderRecord(BaseModel):
-    id: int = 0
-    object_name: str = ""
-    ra: float = 0
-    dec: float = 0
-    version: str = ""
-
 @dataclasses.dataclass
 class Level0Record(BaseModel):
     id: int = 0
@@ -38,6 +30,7 @@ class Level0Record(BaseModel):
     prc_status: int = 0
     prc_time: str=""
     create_time: str=""
+    header: Dict[str,object] = default_field({})
 
 @dataclasses.dataclass
 class CalMergeRecord(BaseModel):

csst_dfs_commons/models/msc.py

@@ -30,6 +30,7 @@ class Level0Record(BaseModel):
     prc_status: int = 0
     prc_time: str=""
     create_time: str=""
+    header: Dict[str,object] = default_field({})
 
 @dataclasses.dataclass
 class CalMergeRecord(BaseModel):
@@ -66,6 +67,17 @@ class Level1Record(BaseModel):
     pipeline_id: str=""
     refs: Dict[str,int] = default_field({})
 
+@dataclasses.dataclass
+class Level1PrcRecord(BaseModel):
+    id: int = 0
+    level1_id: int = 0
+    pipeline_id: str = ""
+    prc_module: str = ""
+    params_file_path: str=""
+    prc_status: int = 0
+    prc_time: str=""
+    result_file_path: str=""  
+
 @dataclasses.dataclass
 class Level2Record(BaseModel):
     id: int = 0
@@ -78,6 +90,20 @@ class Level2Record(BaseModel):
     prc_status: int = 0
     prc_time: str=""
     create_time: str=""
+    import_status: int = 0
+
+@dataclasses.dataclass
+class Level2CoRecord(BaseModel):
+    id: int = 0
+    data_type: str=""
+    filename : str=""
+    file_path: str=""
+    qc2_status: int = 0
+    qc2_time: str=""
+    prc_status: int = 0
+    prc_time: str=""
+    create_time: str=""
+    import_status: int = 0
 
 @dataclasses.dataclass
 class Level2CatalogRecord(BaseModel):
@@ -243,12 +269,164 @@ class Level2CatalogRecord(BaseModel):
     create_time: str = ''
 
 @dataclasses.dataclass
-class Level1PrcRecord(BaseModel):
-    id: int = 0
-    level1_id: int = 0
-    pipeline_id: str = ""
-    prc_module: str = ""
-    params_file_path: str=""
-    prc_status: int = 0
-    prc_time: str=""
-    result_file_path: str=""    
+class Level2CoCatalogRecord(BaseModel):
+    level2_id: int = 0
+    seq: int = 0
+    flux_aper: list = dataclasses.field(default_factory=list)
+    fluxerr_aper: list = dataclasses.field(default_factory=list)
+    mag_aper: list = dataclasses.field(default_factory=list)
+    magerr_aper: list = dataclasses.field(default_factory=list)
+    flux_auto: float = -1
+    fluxerr_auto: float = -1
+    mag_auto: float = -1
+    magerr_auto: float = -1
+    kron_radius: float = -1
+    background: float = -1
+    x_image: float = -1
+    y_image: float = -1
+    alpha_j2000: float = -1
+    delta_j2000: float = -1
+    a_image: float = -1
+    b_image: float = -1
+    theta_image: float = -1
+    a_world: float = -1
+    b_world: float = -1
+    theta_world: float = -1
+    theta_j2000: float = -1
+    errx2_image: float = -1
+    erry2_image: float = -1
+    erra_image: float = -1
+    errb_image: float = -1
+    errtheta_image: float = -1
+    erra_world: float = -1
+    errb_world: float = -1
+    errtheta_world: float = -1
+    errtheta_j2000: float = -1
+    xwin_image: float = -1
+    ywin_image: float = -1
+    alphawin_j2000: float = -1
+    deltawin_j2000: float = -1
+    errx2win_image: float = -1
+    erry2win_image: float = -1
+    flags: int = 0
+    flags_weight: int = 0
+    imaflags_iso: float = -1
+    nimaflags_iso: float = -1
+    fwhm_image: float = -1
+    fwhm_world: float = -1
+    elongation: float = -1
+    ellipticity: float = -1
+    class_star: float = -1
+    flux_radius: float = -1
+    fwhmpsf_image: float = -1
+    fwhmpsf_world: float = -1
+    xpsf_image: float = -1
+    ypsf_image: float = -1
+    alphapsf_j2000: float = -1
+    deltapsf_j2000: float = -1
+    flux_psf: float = -1
+    fluxerr_psf: float = -1
+    mag_psf: float = -1
+    magerr_psf: float = -1
+    niter_psf: int = 0
+    chi2_psf: float = -1
+    errx2psf_image: float = -1
+    erry2psf_image: float = -1
+    chi2_model: float = -1
+    flags_model: int = 0
+    niter_model: int = 0
+    flux_model: float = -1
+    fluxerr_model: float = -1
+    mag_model: float = -1
+    magerr_model: float = -1
+    flux_hybrid: float = -1
+    fluxerr_hybrid: float = -1
+    mag_hybrid: float = -1
+    magerr_hybrid: float = -1
+    flux_max_model: float = -1
+    mu_max_model: float = -1
+    flux_eff_model: float = -1
+    mu_eff_model: float = -1
+    flux_mean_model: float = -1
+    mu_mean_model: float = -1
+    xmodel_image: float = -1
+    ymodel_image: float = -1
+    alphamodel_j2000: float = -1
+    deltamodel_j2000: float = -1
+    erry2model_image: float = -1
+    erramodel_image: float = -1
+    errbmodel_image: float = -1
+    errthetamodel_image: float = -1
+    erramodel_world: float = -1
+    errbmodel_world: float = -1
+    errthetamodel_world: float = -1
+    errthetamodel_j2000: float = -1
+    amodel_image: float = -1
+    bmodel_image: float = -1
+    thetamodel_image: float = -1
+    amodel_world: float = -1
+    bmodel_world: float = -1
+    thetamodel_world: float = -1
+    thetamodel_j2000: float = -1
+    spread_model: float = -1
+    spreaderr_model: float = -1
+    noisearea_model: float = -1
+    flux_spheroid: float = -1
+    fluxerr_spheroid: float = -1
+    mag_spheroid: float = -1
+    magerr_spheroid: float = -1
+    flux_max_spheroid: float = -1
+    mu_max_spheroid: float = -1
+    flux_eff_spheroid: float = -1
+    mu_eff_spheroid: float = -1
+    flux_mean_spheroid: float = -1
+    mu_mean_spheroid: float = -1
+    fluxratio_spheroid: float = -1
+    fluxratioerr_spheroid: float = -1
+    spheroid_reff_image: float = -1
+    spheroid_refferr_image: float = -1
+    spheroid_reff_world: float = -1
+    spheroid_refferr_world: float = -1
+    spheroid_aspect_image: float = -1
+    spheroid_aspecterr_image: float = -1
+    spheroid_aspect_world: float = -1
+    spheroid_aspecterr_world: float = -1
+    spheroid_theta_image: float = -1
+    spheroid_thetaerr_image: float = -1
+    spheroid_theta_world: float = -1
+    spheroid_thetaerr_world: float = -1
+    spheroid_theta_j2000: float = -1
+    spheroid_sersicn: float = -1
+    spheroid_sersicnerr: float = -1
+    flux_disk: float = -1
+    fluxerr_disk: float = -1
+    mag_disk: float = -1
+    magerr_disk: float = -1
+    flux_max_disk: float = -1
+    mu_max_disk: float = -1
+    flux_eff_disk: float = -1
+    mu_eff_disk: float = -1
+    flux_mean_disk: float = -1
+    mu_mean_disk: float = -1
+    fluxratio_disk: float = -1
+    fluxratioerr_disk: float = -1
+    disk_scale_image: float = -1
+    disk_scaleerr_image: float = -1
+    disk_scale_world: float = -1
+    disk_scaleerr_world: float = -1
+    disk_aspect_image: float = -1
+    disk_aspecterr_image: float = -1
+    disk_aspect_world: float = -1
+    disk_aspecterr_world: float = -1
+    disk_inclination: float = -1
+    disk_inclinationerr: float = -1
+    disk_theta_image: float = -1
+    disk_thetaerr_image: float = -1
+    disk_theta_world: float = -1
+    disk_thetaerr_world: float = -1
+    disk_theta_j2000: float = -1    
+    NS8HIdx: int = 0
+    NS16HIdx: int = 0
+    NS32HIdx: int = 0
+    NS64HIdx: int = 0
+    create_time: str = ''

csst_dfs_commons/models/sls.py

@@ -30,6 +30,7 @@ class Level0Record(BaseModel):
     prc_status: int = 0
     prc_time: str=""
     create_time: str=""
+    header: Dict[str,object] = default_field({})
 
 @dataclasses.dataclass
 class CalMergeRecord(BaseModel):
@@ -48,6 +49,7 @@ class CalMergeRecord(BaseModel):
     prc_time: str=""
     create_time: str=""
     level0_ids: list = dataclasses.field(default_factory=list)
+
 @dataclasses.dataclass
 class Level1Record(BaseModel):
     id: int = 0
@@ -63,6 +65,7 @@ class Level1Record(BaseModel):
     create_time: str=""
     pipeline_id: str=""
     refs: Dict[str,int] = default_field({})
+    
 @dataclasses.dataclass
 class Level2Spectra(BaseModel):
     id: int = 0