Skip to content

Commit 259e5fa

Browse files
chadellchadell
committed
Fix and extend testing and code fixes
1 parent 52f579c commit 259e5fa

File tree

8 files changed

+980
-617
lines changed

8 files changed

+980
-617
lines changed

‎diffsync/__init__.py

+27-16
Original file line numberDiff line numberDiff line change
@@ -14,9 +14,8 @@
1414
See the License for the specific language governing permissions and
1515
limitations under the License.
1616
"""
17-
from collections import defaultdict
1817
from inspect import isclass
19-
from typing import Callable, ClassVar, Dict, List, Mapping, MutableMapping, Optional, Text, Tuple, Type, Union
18+
from typing import Callable, ClassVar, Dict, List, Mapping, Optional, Text, Tuple, Type, Union
2019

2120
from pydantic import BaseModel, PrivateAttr
2221
import structlog # type: ignore
@@ -25,7 +24,8 @@
2524
from .enum import DiffSyncModelFlags, DiffSyncFlags, DiffSyncStatus
2625
from .exceptions import ObjectAlreadyExists, ObjectStoreWrongType, ObjectNotFound
2726
from .helpers import DiffSyncDiffer, DiffSyncSyncer
28-
from .store import BaseStore, LocalStore
27+
from .store import BaseStore
28+
from .store.local import LocalStore
2929

3030

3131
class DiffSyncModel(BaseModel):
@@ -370,20 +370,12 @@ class DiffSync:
370370
top_level: ClassVar[List[str]] = []
371371
"""List of top-level modelnames to begin from when diffing or synchronizing."""
372372

373-
_data: MutableMapping[str, MutableMapping[str, DiffSyncModel]]
374-
"""Defaultdict storing model instances.
375-
376-
`self._data[modelname][unique_id] == model_instance`
377-
"""
378-
379373
def __init__(self, name=None, internal_storage_engine=LocalStore):
380374
"""Generic initialization function.
381375
382376
Subclasses should be careful to call super().__init__() if they override this method.
383377
"""
384378

385-
self._log = structlog.get_logger().new(diffsync=self)
386-
387379
if isinstance(internal_storage_engine, BaseStore):
388380
self.store = internal_storage_engine
389381
self.store.diffsync = self
@@ -436,10 +428,10 @@ def load(self):
436428
def dict(self, exclude_defaults: bool = True, **kwargs) -> Mapping:
437429
"""Represent the DiffSync contents as a dict, as if it were a Pydantic model."""
438430
data: Dict[str, Dict[str, Dict]] = {}
439-
for modelname in self._data:
431+
for modelname in self.store.get_all_model_names():
440432
data[modelname] = {}
441-
for unique_id, model in self._data[modelname].items():
442-
data[modelname][unique_id] = model.dict(exclude_defaults=exclude_defaults, **kwargs)
433+
for obj in self.store.get_all(modelname):
434+
data[obj.get_type()][obj.get_unique_id()] = obj.dict(exclude_defaults=exclude_defaults, **kwargs)
443435
return data
444436

445437
def str(self, indent: int = 0) -> str:
@@ -571,6 +563,14 @@ def diff_to(
571563
# Object Storage Management
572564
# ------------------------------------------------------------------------------
573565

566+
def get_all_model_names(self):
567+
"""Get all model names.
568+
569+
Returns:
570+
List[str]: List of model names
571+
"""
572+
return self.store.get_all_model_names()
573+
574574
def get(
575575
self, obj: Union[Text, DiffSyncModel, Type[DiffSyncModel]], identifier: Union[Text, Mapping]
576576
) -> DiffSyncModel:
@@ -659,7 +659,7 @@ def get_or_instantiate(
659659
Returns:
660660
Tuple[DiffSyncModel, bool]: Provides the existing or new object and whether it was created or not.
661661
"""
662-
return self.store.get_or_instantiate(modle=model, ids=ids, attrs=attrs)
662+
return self.store.get_or_instantiate(model=model, ids=ids, attrs=attrs)
663663

664664
def update_or_instantiate(self, model: Type[DiffSyncModel], ids: Dict, attrs: Dict) -> Tuple[DiffSyncModel, bool]:
665665
"""Attempt to update an existing object with provided ids/attrs or instantiate it with provided identifiers and attrs.
@@ -672,7 +672,18 @@ def update_or_instantiate(self, model: Type[DiffSyncModel], ids: Dict, attrs: Di
672672
Returns:
673673
Tuple[DiffSyncModel, bool]: Provides the existing or new object and whether it was created or not.
674674
"""
675-
return self.store.update_or_instantiate(modle=model, ids=ids, attrs=attrs)
675+
return self.store.update_or_instantiate(model=model, ids=ids, attrs=attrs)
676+
677+
def count(self, modelname=None):
678+
"""Count how many objects of one model type exist in the backend store.
679+
680+
Args:
681+
modelname (str): The model name to check the number of elements. If not provided, default to all.
682+
683+
Returns:
684+
Int: Number of elements of the model type
685+
"""
686+
return self.store.count(modelname=modelname)
676687

677688

678689
# DiffSyncModel references DiffSync and DiffSync references DiffSyncModel. Break the typing loop:

‎diffsync/store/__init__.py

+82-170
Original file line numberDiff line numberDiff line change
@@ -1,44 +1,111 @@
1-
from collections import defaultdict
2-
from typing import Callable, ClassVar, Dict, List, Mapping, MutableMapping, Optional, Text, Tuple, Type, Union
1+
"""BaseStore module."""
2+
from typing import Dict, List, Mapping, Text, Tuple, Type, Union
3+
import structlog # type: ignore
34

4-
from diffsync.exceptions import ObjectNotFound, ObjectAlreadyExists
5+
from diffsync.exceptions import ObjectNotFound
6+
7+
# TODO: find proper annotation for "DiffSyncModel" noqa: F821
58

69

710
class BaseStore:
11+
"""Reference store to be implemented in different backends."""
812

9-
def __init__(self, diffsync=None, name=None, *args, **kwargs) -> None:
13+
def __init__(self, diffsync=None, name=None, *args, **kwargs) -> None: # pylint: disable=unused-argument
14+
"""Init method for BaseStore."""
1015
self.diffsync = diffsync
1116
self.name = name if name else self.__class__.__name__
17+
self._log = structlog.get_logger().new(diffsync=self)
1218

1319
def __str__(self):
20+
"""Render store name."""
1421
return self.name
1522

16-
def get(self, obj: Union[Text, "DiffSyncModel", Type["DiffSyncModel"]], identifier: Union[Text, Mapping]):
23+
def get_all_model_names(self):
24+
"""Get all the model names stored.
25+
26+
Return:
27+
List[str]: List of all the model names.
28+
"""
29+
raise NotImplementedError
30+
31+
def get(
32+
self, obj: Union[Text, "DiffSyncModel", Type["DiffSyncModel"]], identifier: Union[Text, Mapping] # noqa: F821
33+
):
34+
"""Get one object from the data store based on its unique id.
35+
36+
Args:
37+
obj: DiffSyncModel class or instance, or modelname string, that defines the type of the object to retrieve
38+
identifier: Unique ID of the object to retrieve, or dict of unique identifier keys/values
39+
40+
Raises:
41+
ValueError: if obj is a str and identifier is a dict (can't convert dict into a uid str without a model class)
42+
ObjectNotFound: if the requested object is not present
43+
"""
1744
raise NotImplementedError
1845

19-
def get_all(self, obj: Union[Text, "DiffSyncModel", Type["DiffSyncModel"]]) -> List["DiffSyncModel"]:
46+
def get_all(self, obj: Union[Text, "DiffSyncModel", Type["DiffSyncModel"]]) -> List["DiffSyncModel"]: # noqa: F821
47+
"""Get all objects of a given type.
48+
49+
Args:
50+
obj: DiffSyncModel class or instance, or modelname string, that defines the type of the objects to retrieve
51+
52+
Returns:
53+
List[DiffSyncModel]: List of Object
54+
"""
2055
raise NotImplementedError
2156

2257
def get_by_uids(
23-
self, uids: List[Text], obj: Union[Text, "DiffSyncModel", Type["DiffSyncModel"]]
24-
) -> List["DiffSyncModel"]:
58+
self, uids: List[Text], obj: Union[Text, "DiffSyncModel", Type["DiffSyncModel"]] # noqa: F821
59+
) -> List["DiffSyncModel"]: # noqa: F821
60+
"""Get multiple objects from the store by their unique IDs/Keys and type.
61+
62+
Args:
63+
uids: List of unique id / key identifying object in the database.
64+
obj: DiffSyncModel class or instance, or modelname string, that defines the type of the objects to retrieve
65+
66+
Raises:
67+
ObjectNotFound: if any of the requested UIDs are not found in the store
68+
"""
2569
raise NotImplementedError
2670

27-
def remove(self, obj: "DiffSyncModel", remove_children: bool = False):
71+
def remove(self, obj: "DiffSyncModel", remove_children: bool = False): # noqa: F821
72+
"""Remove a DiffSyncModel object from the store.
73+
74+
Args:
75+
obj (DiffSyncModel): object to remove
76+
remove_children (bool): If True, also recursively remove any children of this object
77+
78+
Raises:
79+
ObjectNotFound: if the object is not present
80+
"""
2881
raise NotImplementedError
2982

30-
def add(self, obj: "DiffSyncModel"):
83+
def add(self, obj: "DiffSyncModel"): # noqa: F821
84+
"""Add a DiffSyncModel object to the store.
85+
86+
Args:
87+
obj (DiffSyncModel): Object to store
88+
89+
Raises:
90+
ObjectAlreadyExists: if a different object with the same uid is already present.
91+
"""
3192
raise NotImplementedError
3293

33-
def update(self, obj: "DiffSyncModel"):
94+
def update(self, obj: "DiffSyncModel"): # noqa: F821
95+
"""Update a DiffSyncModel object to the store.
96+
97+
Args:
98+
obj (DiffSyncModel): Object to update
99+
"""
34100
raise NotImplementedError
35101

36102
def count(self, modelname):
103+
"""Returns the number of elements of an specific model name."""
37104
raise NotImplementedError
38105

39106
def get_or_instantiate(
40-
self, model: Type["DiffSyncModel"], ids: Dict, attrs: Dict = None
41-
) -> Tuple["DiffSyncModel", bool]:
107+
self, model: Type["DiffSyncModel"], ids: Dict, attrs: Dict = None # noqa: F821
108+
) -> Tuple["DiffSyncModel", bool]: # noqa: F821
42109
"""Attempt to get the object with provided identifiers or instantiate it with provided identifiers and attrs.
43110
44111
Args:
@@ -63,8 +130,8 @@ def get_or_instantiate(
63130
return obj, created
64131

65132
def update_or_instantiate(
66-
self, model: Type["DiffSyncModel"], ids: Dict, attrs: Dict
67-
) -> Tuple["DiffSyncModel", bool]:
133+
self, model: Type["DiffSyncModel"], ids: Dict, attrs: Dict # noqa: F821
134+
) -> Tuple["DiffSyncModel", bool]: # noqa: F821
68135
"""Attempt to update an existing object with provided ids/attrs or instantiate it with provided identifiers and attrs.
69136
70137
Args:
@@ -90,158 +157,3 @@ def update_or_instantiate(
90157
setattr(obj, attr, value)
91158

92159
return obj, created
93-
94-
95-
class LocalStore(BaseStore):
96-
97-
def __init__(self, *args, **kwargs) -> None:
98-
super().__init__(*args, **kwargs)
99-
100-
self._data = defaultdict(dict)
101-
102-
def get(self, obj: Union[Text, "DiffSyncModel", Type["DiffSyncModel"]], identifier: Union[Text, Mapping]):
103-
"""Get one object from the data store based on its unique id.
104-
105-
Args:
106-
obj: DiffSyncModel class or instance, or modelname string, that defines the type of the object to retrieve
107-
identifier: Unique ID of the object to retrieve, or dict of unique identifier keys/values
108-
109-
Raises:
110-
ValueError: if obj is a str and identifier is a dict (can't convert dict into a uid str without a model class)
111-
ObjectNotFound: if the requested object is not present
112-
"""
113-
if isinstance(obj, str):
114-
modelname = obj
115-
if not hasattr(self, obj):
116-
object_class = None
117-
else:
118-
object_class = getattr(self, obj)
119-
else:
120-
object_class = obj
121-
modelname = obj.get_type()
122-
123-
if isinstance(identifier, str):
124-
uid = identifier
125-
elif object_class:
126-
uid = object_class.create_unique_id(**identifier)
127-
else:
128-
raise ValueError(
129-
f"Invalid args: ({obj}, {identifier}): "
130-
f"either {obj} should be a class/instance or {identifier} should be a str"
131-
)
132-
133-
if uid not in self._data[modelname]:
134-
raise ObjectNotFound(f"{modelname} {uid} not present in {str(self)}")
135-
return self._data[modelname][uid]
136-
137-
def get_all(self, obj: Union[Text, "DiffSyncModel", Type["DiffSyncModel"]]) -> List["DiffSyncModel"]:
138-
"""Get all objects of a given type.
139-
140-
Args:
141-
obj: DiffSyncModel class or instance, or modelname string, that defines the type of the objects to retrieve
142-
143-
Returns:
144-
List[DiffSyncModel]: List of Object
145-
"""
146-
if isinstance(obj, str):
147-
modelname = obj
148-
else:
149-
modelname = obj.get_type()
150-
151-
return list(self._data[modelname].values())
152-
153-
def get_by_uids(
154-
self, uids: List[Text], obj: Union[Text, "DiffSyncModel", Type["DiffSyncModel"]]
155-
) -> List["DiffSyncModel"]:
156-
"""Get multiple objects from the store by their unique IDs/Keys and type.
157-
158-
Args:
159-
uids: List of unique id / key identifying object in the database.
160-
obj: DiffSyncModel class or instance, or modelname string, that defines the type of the objects to retrieve
161-
162-
Raises:
163-
ObjectNotFound: if any of the requested UIDs are not found in the store
164-
"""
165-
if isinstance(obj, str):
166-
modelname = obj
167-
else:
168-
modelname = obj.get_type()
169-
170-
results = []
171-
for uid in uids:
172-
if uid not in self._data[modelname]:
173-
raise ObjectNotFound(f"{modelname} {uid} not present in {str(self)}")
174-
results.append(self._data[modelname][uid])
175-
return results
176-
177-
def add(self, obj: "DiffSyncModel"):
178-
"""Add a DiffSyncModel object to the store.
179-
180-
Args:
181-
obj (DiffSyncModel): Object to store
182-
183-
Raises:
184-
ObjectAlreadyExists: if a different object with the same uid is already present.
185-
"""
186-
modelname = obj.get_type()
187-
uid = obj.get_unique_id()
188-
189-
existing_obj = self._data[modelname].get(uid)
190-
if existing_obj:
191-
if existing_obj is not obj:
192-
raise ObjectAlreadyExists(f"Object {uid} already present", obj)
193-
# Return so we don't have to change anything on the existing object and underlying data
194-
return
195-
196-
if not obj.diffsync:
197-
obj.diffsync = self.diffsync
198-
199-
self._data[modelname][uid] = obj
200-
201-
def update(self, obj: "DiffSyncModel"):
202-
modelname = obj.get_type()
203-
uid = obj.get_unique_id()
204-
205-
existing_obj = self._data[modelname].get(uid)
206-
if existing_obj is obj:
207-
return
208-
209-
self._data[modelname][uid] = obj
210-
211-
def remove(self, obj: "DiffSyncModel", remove_children: bool = False):
212-
"""Remove a DiffSyncModel object from the store.
213-
214-
Args:
215-
obj (DiffSyncModel): object to remove
216-
remove_children (bool): If True, also recursively remove any children of this object
217-
218-
Raises:
219-
ObjectNotFound: if the object is not present
220-
"""
221-
modelname = obj.get_type()
222-
uid = obj.get_unique_id()
223-
224-
if uid not in self._data[modelname]:
225-
raise ObjectNotFound(f"{modelname} {uid} not present in {str(self)}")
226-
227-
if obj.diffsync:
228-
obj.diffsync = None
229-
230-
del self._data[modelname][uid]
231-
232-
if remove_children:
233-
for child_type, child_fieldname in obj.get_children_mapping().items():
234-
for child_id in getattr(obj, child_fieldname):
235-
try:
236-
child_obj = self.get(child_type, child_id)
237-
self.remove(child_obj, remove_children=remove_children)
238-
except ObjectNotFound:
239-
pass
240-
# Since this is "cleanup" code, log an error and continue, instead of letting the exception raise
241-
# self._log.error(f"Unable to remove child {child_id} of {modelname} {uid} - not found!")
242-
243-
def count(self, modelname=None):
244-
if not modelname:
245-
return sum(len(entries) for entries in self._data.values())
246-
else:
247-
return len(self._data[modelname])

0 commit comments

Comments
 (0)