Skip to content

Commit cae2fd5

Browse files
caffixcaffix
committed
deduplication improvements for OAM Organizations
1 parent 9894967 commit cae2fd5

File tree

4 files changed

+67
-41
lines changed

4 files changed

+67
-41
lines changed

‎engine/plugins/enrich/tls_cert.go‎

Lines changed: 4 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -371,22 +371,13 @@ func (te *tlsexpand) storeContact(e *et.Event, c *tlsContact, asset *dbt.Entity,
371371
}
372372
}
373373
}
374-
if len(ct.Organization) > 0 && ct.Organization[0] != "" && m.IsMatch(string(oam.Organization)) {
375-
if a, err := support.CreateOrgAsset(e.Session, &org.Organization{
374+
if m.IsMatch(string(oam.Organization)) && len(ct.Organization) > 0 && ct.Organization[0] != "" {
375+
o := &org.Organization{
376376
ID: uuid.New().String(),
377377
Name: ct.Organization[0],
378-
}, src); err == nil && a != nil {
379-
if edge, err := e.Session.Cache().CreateEdge(&dbt.Edge{
380-
Relation: &general.SimpleRelation{Name: "organization"},
381-
FromEntity: cr,
382-
ToEntity: a,
383-
}); err == nil && edge != nil {
384-
_, _ = e.Session.Cache().CreateEdgeProperty(edge, &general.SourceProperty{
385-
Source: src.Name,
386-
Confidence: src.Confidence,
387-
})
388-
}
389378
}
379+
380+
_, _ = support.CreateOrgAsset(e.Session, cr, &general.SimpleRelation{Name: "organization"}, o, src)
390381
}
391382
if len(ct.OrganizationalUnit) > 0 && ct.OrganizationalUnit[0] != "" && m.IsMatch(string(oam.URL)) {
392383
if u := support.ExtractURLFromString(ct.OrganizationalUnit[0]); u != nil {

‎engine/plugins/rdap/plugin.go‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -243,14 +243,13 @@ func (rd *rdapPlugin) storeEntity(e *et.Event, level int, entity *rdap.Entity, a
243243
_ = rd.createContactEdge(e.Session, cr, a, &general.SimpleRelation{Name: "person"}, src)
244244
}
245245
}
246-
} else if m.IsMatch(string(oam.Organization)) && name != "" &&
247-
!support.OrgNameExistsInContactRecord(e.Session, cr, name) {
248-
if a, err := support.CreateOrgAsset(e.Session, &org.Organization{
246+
} else if m.IsMatch(string(oam.Organization)) {
247+
o := &org.Organization{
249248
ID: uuid.New().String(),
250249
Name: name,
251-
}, src); err == nil && a != nil {
252-
_ = rd.createContactEdge(e.Session, cr, a, &general.SimpleRelation{Name: "organization"}, src)
253250
}
251+
252+
_, _ = support.CreateOrgAsset(e.Session, cr, &general.SimpleRelation{Name: "organization"}, o, src)
254253
}
255254
if adr := v.GetFirst("adr"); adr != nil && m.IsMatch(string(oam.Location)) {
256255
if label, ok := adr.Parameters["label"]; ok {

‎engine/plugins/support/database.go‎

Lines changed: 54 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ import (
1818
dbt "github.com/owasp-amass/asset-db/types"
1919
oam "github.com/owasp-amass/open-asset-model"
2020
oamcert "github.com/owasp-amass/open-asset-model/certificate"
21+
"github.com/owasp-amass/open-asset-model/contact"
2122
oamdns "github.com/owasp-amass/open-asset-model/dns"
2223
"github.com/owasp-amass/open-asset-model/general"
2324
"github.com/owasp-amass/open-asset-model/org"
@@ -235,33 +236,49 @@ func CreateServiceAsset(session et.Session, src *dbt.Entity, rel oam.Relation, s
235236
return result, err
236237
}
237238

238-
func CreateOrgAsset(session et.Session, o *org.Organization, src *et.Source) (*dbt.Entity, error) {
239+
func CreateOrgAsset(session et.Session, obj *dbt.Entity, rel oam.Relation, o *org.Organization, src *et.Source) (*dbt.Entity, error) {
240+
if orgent := orgDedupChecks(session, obj, rel, o); orgent != nil {
241+
if err := createRelation(session, obj, rel, orgent, src); err != nil {
242+
return nil, err
243+
}
244+
return orgent, nil
245+
}
246+
239247
id := &general.Identifier{
240248
UniqueID: fmt.Sprintf("%s:%s", "org_name", o.Name),
241249
EntityID: o.Name,
242250
Type: "org_name",
243251
}
244252

245253
if ident, err := session.Cache().CreateAsset(id); err == nil && ident != nil {
246-
if o, err := session.Cache().CreateAsset(o); err == nil && o != nil {
247-
if e, err := session.Cache().CreateEdge(&dbt.Edge{
248-
Relation: &general.SimpleRelation{Name: "id"},
249-
FromEntity: o,
250-
ToEntity: ident,
251-
}); err == nil && e != nil {
252-
_, err = session.Cache().CreateEdgeProperty(e, &general.SourceProperty{
253-
Source: src.Name,
254-
Confidence: src.Confidence,
255-
})
256-
return o, err
254+
if orgent, err := session.Cache().CreateAsset(o); err == nil && orgent != nil {
255+
if err := createRelation(session, orgent, &general.SimpleRelation{Name: "id"}, ident, src); err != nil {
256+
return nil, err
257257
}
258+
if err := createRelation(session, obj, rel, orgent, src); err != nil {
259+
return nil, err
260+
}
261+
return orgent, nil
258262
}
259263
}
260264

261265
return nil, errors.New("failed to create the OAM Organization asset")
262266
}
263267

264-
func OrgHasName(session et.Session, org *dbt.Entity, name string) bool {
268+
func orgDedupChecks(session et.Session, obj *dbt.Entity, rel oam.Relation, o *org.Organization) *dbt.Entity {
269+
var result *dbt.Entity
270+
271+
switch obj.Asset.(type) {
272+
case *contact.ContactRecord:
273+
if org, found := orgNameExistsInContactRecord(session, obj, o.Name); found {
274+
result = org
275+
}
276+
}
277+
278+
return result
279+
}
280+
281+
func orgHasName(session et.Session, org *dbt.Entity, name string) bool {
265282
if org == nil {
266283
return false
267284
}
@@ -278,19 +295,38 @@ func OrgHasName(session et.Session, org *dbt.Entity, name string) bool {
278295
return false
279296
}
280297

281-
func OrgNameExistsInContactRecord(session et.Session, cr *dbt.Entity, name string) bool {
298+
func orgNameExistsInContactRecord(session et.Session, cr *dbt.Entity, name string) (*dbt.Entity, bool) {
282299
if cr == nil {
283-
return false
300+
return nil, false
284301
}
285302

286303
if edges, err := session.Cache().OutgoingEdges(cr, time.Time{}, "organization"); err == nil && len(edges) > 0 {
287304
for _, edge := range edges {
288305
if a, err := session.Cache().FindEntityById(edge.ToEntity.ID); err == nil && a != nil {
289-
if _, ok := a.Asset.(*org.Organization); ok && OrgHasName(session, a, name) {
290-
return true
306+
if _, ok := a.Asset.(*org.Organization); ok && orgHasName(session, a, name) {
307+
return a, true
291308
}
292309
}
293310
}
294311
}
295-
return false
312+
return nil, false
313+
}
314+
315+
func createRelation(session et.Session, obj *dbt.Entity, rel oam.Relation, subject *dbt.Entity, src *et.Source) error {
316+
edge, err := session.Cache().CreateEdge(&dbt.Edge{
317+
Relation: rel,
318+
FromEntity: obj,
319+
ToEntity: subject,
320+
})
321+
if err != nil {
322+
return err
323+
} else if edge == nil {
324+
return errors.New("failed to create the edge")
325+
}
326+
327+
_, err = session.Cache().CreateEdgeProperty(edge, &general.SourceProperty{
328+
Source: src.Name,
329+
Confidence: src.Confidence,
330+
})
331+
return err
296332
}

‎engine/plugins/whois/domain_record.go‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -224,14 +224,14 @@ func (r *domrec) storeContact(e *et.Event, c *domrecContact, dr *dbt.Entity, m *
224224
}
225225
}
226226
}
227-
if m.IsMatch(string(oam.Organization)) && wc.Organization != "" &&
228-
!support.OrgNameExistsInContactRecord(e.Session, cr, wc.Organization) {
229-
if a, err := support.CreateOrgAsset(e.Session, &org.Organization{
227+
if m.IsMatch(string(oam.Organization)) {
228+
o := &org.Organization{
230229
ID: uuid.New().String(),
231230
Name: wc.Organization,
232-
}, r.plugin.source); err == nil && a != nil {
233-
r.createSimpleEdge(e.Session.Cache(), &general.SimpleRelation{Name: "organization"}, cr, a)
234231
}
232+
233+
_, _ = support.CreateOrgAsset(e.Session, cr,
234+
&general.SimpleRelation{Name: "organization"}, o, r.plugin.source)
235235
}
236236
if loc := support.StreetAddressToLocation(addr); loc != nil {
237237
if a, err := e.Session.Cache().CreateAsset(loc); err == nil && a != nil {

0 commit comments

Comments
 (0)