Skip to content

Commit 75f9008

Browse files
caffixcaffix
committed
additional development on the GLEIF plugin
1 parent 0b4df31 commit 75f9008

File tree

3 files changed

+142
-15
lines changed

3 files changed

+142
-15
lines changed

‎engine/plugins/api/gleif/fuzzy.go‎

Lines changed: 82 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import (
99
"encoding/json"
1010
"errors"
1111
"fmt"
12+
"log/slog"
1213
"net/url"
1314
"strings"
1415
"time"
@@ -18,6 +19,7 @@ import (
1819
"github.com/owasp-amass/amass/v4/utils/net/http"
1920
dbt "github.com/owasp-amass/asset-db/types"
2021
oam "github.com/owasp-amass/open-asset-model"
22+
"github.com/owasp-amass/open-asset-model/contact"
2123
"github.com/owasp-amass/open-asset-model/general"
2224
"github.com/owasp-amass/open-asset-model/org"
2325
)
@@ -116,23 +118,100 @@ func (fc *fuzzyCompletions) query(e *et.Event, orgent *dbt.Entity, src *et.Sourc
116118
EntityID: d.Relationships.LEIRecords.Data.ID,
117119
Type: general.LEICode,
118120
}
121+
break
119122
}
120123
}
121124
if lei == nil {
122125
return nil
123126
}
124127

125128
rec, err := fc.plugin.getLEIRecord(e, lei, src)
126-
if err == nil {
129+
if err == nil && fc.locMatch(e, orgent, rec) {
127130
return nil
128131
}
129132

130-
return fc.store(e, orgent, lei, fc.plugin.source)
133+
return fc.store(e, lei, src)
131134
}
132135

133-
func (fc *fuzzyCompletions) store(e *et.Event, orgent *dbt.Entity, id *general.Identifier, src *et.Source) *dbt.Entity {
136+
func (fc *fuzzyCompletions) locMatch(e *et.Event, orgent *dbt.Entity, rec *leiRecord) bool {
137+
if edges, err := e.Session.Cache().OutgoingEdges(orgent, time.Time{}, "location"); err == nil {
138+
for _, edge := range edges {
139+
if a, err := e.Session.Cache().FindEntityById(edge.ToEntity.ID); err == nil && a != nil {
140+
if loc, ok := a.Asset.(*contact.Location); ok &&
141+
(loc.PostalCode == rec.Attributes.Entity.LegalAddress.PostalCode ||
142+
(loc.PostalCode == rec.Attributes.Entity.HeadquartersAddress.PostalCode)) {
143+
return true
144+
}
145+
}
146+
}
147+
}
148+
149+
var crs []*dbt.Entity
150+
if edges, err := e.Session.Cache().IncomingEdges(orgent, time.Time{}, "organization"); err == nil {
151+
for _, edge := range edges {
152+
if a, err := e.Session.Cache().FindEntityById(edge.FromEntity.ID); err == nil && a != nil {
153+
if _, ok := a.Asset.(*contact.ContactRecord); ok {
154+
crs = append(crs, a)
155+
}
156+
}
157+
}
158+
}
159+
160+
for _, cr := range crs {
161+
if edges, err := e.Session.Cache().OutgoingEdges(cr, time.Time{}, "location"); err == nil {
162+
for _, edge := range edges {
163+
if a, err := e.Session.Cache().FindEntityById(edge.ToEntity.ID); err == nil && a != nil {
164+
if loc, ok := a.Asset.(*contact.Location); ok &&
165+
(loc.PostalCode == rec.Attributes.Entity.LegalAddress.PostalCode ||
166+
(loc.PostalCode == rec.Attributes.Entity.HeadquartersAddress.PostalCode)) {
167+
return true
168+
}
169+
}
170+
}
171+
}
172+
}
173+
return false
174+
}
175+
176+
func (fc *fuzzyCompletions) store(e *et.Event, id *general.Identifier, src *et.Source) *dbt.Entity {
177+
a, err := e.Session.Cache().CreateAsset(id)
178+
179+
if err == nil && a != nil {
180+
_, _ = e.Session.Cache().CreateEntityProperty(a, &general.SourceProperty{
181+
Source: src.Name,
182+
Confidence: src.Confidence,
183+
})
184+
return a
185+
}
186+
187+
e.Session.Log().Error(err.Error(), slog.Group("plugin", "name", fc.plugin.name, "handler", fc.name))
134188
return nil
135189
}
136190

137191
func (fc *fuzzyCompletions) process(e *et.Event, orgent, ident *dbt.Entity, src *et.Source) {
192+
edge, err := e.Session.Cache().CreateEdge(&dbt.Edge{
193+
Relation: &general.SimpleRelation{Name: "id"},
194+
FromEntity: orgent,
195+
ToEntity: ident,
196+
})
197+
if err != nil && edge == nil {
198+
e.Session.Log().Error(err.Error(), slog.Group("plugin", "name", fc.plugin.name, "handler", fc.name))
199+
return
200+
}
201+
202+
_, _ = e.Session.Cache().CreateEdgeProperty(edge, &general.SourceProperty{
203+
Source: src.Name,
204+
Confidence: src.Confidence,
205+
})
206+
207+
id := ident.Asset.(*general.Identifier)
208+
_ = e.Dispatcher.DispatchEvent(&et.Event{
209+
Name: id.UniqueID,
210+
Entity: ident,
211+
Session: e.Session,
212+
})
213+
214+
o := orgent.Asset.(*org.Organization)
215+
e.Session.Log().Info("relationship discovered", "from", o.Name, "relation", "id",
216+
"to", id.UniqueID, slog.Group("plugin", "name", fc.plugin.name, "handler", fc.name))
138217
}

‎engine/plugins/api/gleif/plugin.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -228,7 +228,7 @@ type leiRelationshipLinks struct {
228228
ReportingException string `json:"reporting-exception"`
229229
}
230230

231-
func (g *gleif) getLEIRecord(e *et.Event, ident *general.Identifier, src *et.Source) (*singleResponse, error) {
231+
func (g *gleif) getLEIRecord(e *et.Event, ident *general.Identifier, src *et.Source) (*leiRecord, error) {
232232
g.rlimit.Take()
233233

234234
u := "https://api.gleif.org/api/v1/lei-records/" + ident.EntityID
@@ -243,5 +243,5 @@ func (g *gleif) getLEIRecord(e *et.Event, ident *general.Identifier, src *et.Sou
243243
} else if len(result.Data.ID) == 0 {
244244
return nil, errors.New("Failed to find LEI record")
245245
}
246-
return &result, nil
246+
return &result.Data, nil
247247
}

‎engine/plugins/api/gleif/related.go‎

Lines changed: 58 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ import (
1616
dbt "github.com/owasp-amass/asset-db/types"
1717
oam "github.com/owasp-amass/open-asset-model"
1818
"github.com/owasp-amass/open-asset-model/general"
19+
"github.com/owasp-amass/open-asset-model/org"
1920
)
2021

2122
type relatedOrgs struct {
@@ -27,29 +28,76 @@ func (ro *relatedOrgs) check(e *et.Event) error {
2728
ident, ok := e.Entity.Asset.(*general.Identifier)
2829
if !ok {
2930
return errors.New("failed to extract the Identifier asset")
31+
} else if ident.Type != general.LEICode {
32+
return nil
3033
}
3134

32-
since, err := support.TTLStartTime(e.Session.Config(), string(oam.Identifier), string(oam.Identifier), lr.name)
35+
since, err := support.TTLStartTime(e.Session.Config(), string(oam.Identifier), string(oam.Identifier), ro.name)
3336
if err != nil {
3437
return err
3538
}
3639

37-
var names []*dbt.Entity
40+
var orgs []*dbt.Entity
3841
if support.AssetMonitoredWithinTTL(e.Session, e.Entity, ro.plugin.source, since) {
39-
names = append(names, ro.lookup(e, ident, ro.plugin.source, since)...)
42+
orgs = append(orgs, ro.lookup(e, e.Entity, ro.plugin.source, since)...)
4043
} else {
41-
names = append(names, ro.query(e, ident, ro.plugin.source)...)
44+
orgs = append(orgs, ro.query(e, ident, ro.plugin.source)...)
4245
support.MarkAssetMonitored(e.Session, e.Entity, ro.plugin.source)
4346
}
4447

45-
if len(names) > 0 {
46-
ro.process(e, names, ro.plugin.source)
48+
if len(orgs) > 0 {
49+
ro.process(e, orgs, ro.plugin.source)
4750
}
4851
return nil
4952
}
5053

51-
func (ro *relatedOrgs) lookup(e *et.Event, ident *general.Identifier, src *et.Source, since time.Time) []*dbt.Entity {
52-
return support.SourceToAssetsWithinTTL(e.Session, ident.Key(), string(oam.Identifier), ro.plugin.source, since)
54+
func (ro *relatedOrgs) lookup(e *et.Event, ident *dbt.Entity, src *et.Source, since time.Time) []*dbt.Entity {
55+
var o *dbt.Entity
56+
57+
if edges, err := e.Session.Cache().IncomingEdges(ident, since, "id"); err == nil {
58+
for _, edge := range edges {
59+
if tags, err := e.Session.Cache().GetEdgeTags(edge, since, src.Name); err != nil || len(tags) == 0 {
60+
continue
61+
}
62+
if a, err := e.Session.Cache().FindEntityById(edge.FromEntity.ID); err == nil && a != nil {
63+
if _, ok := a.Asset.(*org.Organization); ok {
64+
o = a
65+
break
66+
}
67+
}
68+
}
69+
}
70+
71+
var p *dbt.Entity
72+
if edges, err := e.Session.Cache().OutgoingEdges(o, since, "parent"); err == nil {
73+
for _, edge := range edges {
74+
if tags, err := e.Session.Cache().GetEdgeTags(edge, since, src.Name); err != nil || len(tags) == 0 {
75+
continue
76+
}
77+
if a, err := e.Session.Cache().FindEntityById(edge.ToEntity.ID); err == nil && a != nil {
78+
if _, ok := a.Asset.(*org.Organization); ok {
79+
p = a
80+
break
81+
}
82+
}
83+
}
84+
}
85+
86+
var children []*dbt.Entity
87+
if edges, err := e.Session.Cache().OutgoingEdges(o, since, "subsidiary"); err == nil {
88+
for _, edge := range edges {
89+
if tags, err := e.Session.Cache().GetEdgeTags(edge, since, src.Name); err != nil || len(tags) == 0 {
90+
continue
91+
}
92+
if a, err := e.Session.Cache().FindEntityById(edge.ToEntity.ID); err == nil && a != nil {
93+
if _, ok := a.Asset.(*org.Organization); ok {
94+
children = append(children, a)
95+
}
96+
}
97+
}
98+
}
99+
100+
return append([]*dbt.Entity{o, p}, children...)
53101
}
54102

55103
func (ro *relatedOrgs) query(e *et.Event, ident *general.Identifier, src *et.Source) []*dbt.Entity {
@@ -62,11 +110,11 @@ func (ro *relatedOrgs) query(e *et.Event, ident *general.Identifier, src *et.Sou
62110
}
63111

64112
var result singleResponse
65-
if err := json.Unmarshal([]byte(resp.Body), &result); err != nil || len(result.Data) == 0 {
113+
if err := json.Unmarshal([]byte(resp.Body), &result); err != nil {
66114
return nil
67115
}
68116

69-
return ro.store(e, &result, ro.plugin.source)
117+
return ro.store(e, &result, src)
70118
}
71119

72120
func (ro *relatedOrgs) store(e *et.Event, lei *singleResponse, src *et.Source) []*dbt.Entity {

0 commit comments

Comments
 (0)