From d2cfc60f1d266d98e5b9f63e46c689da99c2d4b2 Mon Sep 17 00:00:00 2001 From: Anderson Banihirwe Date: Tue, 1 Feb 2022 15:49:12 -0700 Subject: [PATCH 1/2] Add `last_updated` to the `ESMCatalogModel` --- intake_esm/cat.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/intake_esm/cat.py b/intake_esm/cat.py index 62ac8c14..55a1f260 100644 --- a/intake_esm/cat.py +++ b/intake_esm/cat.py @@ -1,3 +1,4 @@ +import datetime import enum import json import os @@ -93,6 +94,7 @@ class ESMCatalogModel(pydantic.BaseModel): catalog_file: pydantic.StrictStr = None description: pydantic.StrictStr = None title: pydantic.StrictStr = None + last_updated: typing.Optional[typing.Union[datetime.datetime, datetime.date]] = (None,) _df: typing.Optional[pd.DataFrame] = pydantic.PrivateAttr() class Config: @@ -113,6 +115,8 @@ def validate_catalog(cls, values): def from_dict(cls, data: typing.Dict) -> 'ESMCatalogModel': esmcat = data['esmcat'] df = data['df'] + if 'last_updated' not in esmcat: + esmcat['last_updated'] = None cat = cls.parse_obj(esmcat) cat._df = df return cat @@ -166,6 +170,7 @@ def save( for key in {'catalog_dict', 'catalog_file'}: data.pop(key, None) data['id'] = name + data['last_updated'] = datetime.datetime.now().utcnow().strftime('%Y-%m-%dT%H:%M:%SZ') if catalog_type == 'file': csv_kwargs = {'index': False} @@ -211,7 +216,10 @@ def load( _mapper = fsspec.get_mapper(json_file, **storage_options) with fsspec.open(json_file, **storage_options) as fobj: - cat = cls.parse_raw(fobj.read()) + data = json.loads(fobj.read()) + if 'last_updated' not in data: + data['last_updated'] = None + cat = cls.parse_obj(data) if cat.catalog_file: if _mapper.fs.exists(cat.catalog_file): csv_path = cat.catalog_file From ae4432218874666264ed7f6d0b467c725eb788d3 Mon Sep 17 00:00:00 2001 From: Anderson Banihirwe Date: Tue, 1 Feb 2022 15:50:17 -0700 Subject: [PATCH 2/2] More updates --- intake_esm/cat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/intake_esm/cat.py b/intake_esm/cat.py index 55a1f260..136c619d 100644 --- a/intake_esm/cat.py +++ b/intake_esm/cat.py @@ -94,7 +94,7 @@ class ESMCatalogModel(pydantic.BaseModel): catalog_file: pydantic.StrictStr = None description: pydantic.StrictStr = None title: pydantic.StrictStr = None - last_updated: typing.Optional[typing.Union[datetime.datetime, datetime.date]] = (None,) + last_updated: typing.Optional[typing.Union[datetime.datetime, datetime.date]] = None _df: typing.Optional[pd.DataFrame] = pydantic.PrivateAttr() class Config: