Implement IEP009 - product & vuln. identification

CHANGELOG.md

@@ -21,6 +21,18 @@ Please refer to the [NEWS](NEWS.md) for a list of changes which have an affect o
 
 ### Data Format
 
+- Implementing [IEP009](https://github.com/certtools/ieps/tree/main/009) introducing fields to
+  identify products and vulnerabilities: `product.full_name`, `product.name`, `product.vendor`,
+  `product.version`, `product.vulnerabilities`. To store in existing PostgreSQL instances, a following
+  schema update will be necessary:
+  ```sql
+  ALTER TABLE events ADD "product.full_name" text;
+  ALTER TABLE events ADD "product.name" text;
+  ALTER TABLE events ADD "product.vendor" text;
+  ALTER TABLE events ADD "product.version" text;
+  ALTER TABLE events ADD "product.vulnerabilities" text;
+  ```
+
 ### Bots
 #### Collectors
 

NEWS.md

@@ -18,6 +18,15 @@ Please refer to the change log for a full list of changes.
 ### Tools
 
 ### Data Format
+To save new fields from IntelMQ Data Format in existing PostgreSQL instances, the following schema
+update is necessary:
+```sql
+ALTER TABLE events ADD "product.full_name" text;
+ALTER TABLE events ADD "product.name" text;
+ALTER TABLE events ADD "product.vendor" text;
+ALTER TABLE events ADD "product.version" text;
+ALTER TABLE events ADD "product.vulnerabilities" text;
+```
 
 ### Configuration
 

intelmq/etc/harmonization.conf

@@ -209,6 +209,26 @@
             "description": "Event data converted into foreign format, intended to be exported by output plugin.",
             "type": "JSON"
         },
+        "product.full_name": {
+            "description": "A human readable product name. If a machine-readable format isn't available, this field should be used. It can directly use the version identification strings presented by the product. If not given, a good enough value can usually be constructed by concatenating product.product and product.version, or by consulting external sources such as the CPE Product Dictionary. Example: openssh_/8.9",
+            "type": "String"
+        },
+        "product.name": {
+            "description": "Product name, recommended being as the product in the CPE format. Example: openssh",
+            "type": "LowercaseString"
+        },
+        "product.vendor": {
+            "description": "Vendor name, recommended being as vendor in the CPE format. Example: openbsd",
+            "type": "LowercaseString"
+        },
+        "product.version": {
+            "description": "Product version, recommended being as version in the CPE format. Example: 8.9",
+            "type": "LowercaseString"
+        },
+        "product.vulnerabilities": {
+            "description": "List of vulnerability IDs, separated by semicolons. It's recommended to use a CVE ID where available, and other easily retrievable IDs in other cases, e.g. Github Advisory Database ID. Each vulnerability should only be listed once, and multiple values should be used if there are several different vulnerabilities. However, it's not necessary for a source to list all possible vulnerabilities for a given piece of software. Example: cve-2023-38408;cve-2023-28531;cve-2008-3844;cve-2007-2768",
+            "type": "LowercaseString"
+        },
         "protocol.application": {
             "description": "e.g. vnc, ssh, sip, irc, http or smtp.",
             "length": 100,

intelmq/lib/upgrades.py

@@ -41,7 +41,8 @@
            'v320_update_turris_greylist_url',
            'v322_url_replacement',
            'v322_removed_feeds_and_bots',
-           'v340_deprecations'
+           'v340_deprecations',
+           'v341_new_fields'
            ]
 
 
@@ -974,6 +975,33 @@ def v340_deprecations(configuration, harmonization, dry_run, **kwargs):
     return message or changed, configuration, harmonization
 
 
+def v341_new_fields(configuration, harmonization, dry_run, **kwargs):
+    """
+    Add new fields to IntelMQ Data Format
+    """
+    changed = None
+    if "event" not in harmonization:
+        return changed, configuration, harmonization
+
+    builtin_harmonisation = load_configuration(
+        resource_filename("intelmq", "etc/harmonization.conf")
+    )
+    for field in [
+        "product.full_name",
+        "product.name",
+        "product.vendor",
+        "product.version",
+        "product.vulnerabilities",
+    ]:
+        if field not in harmonization["event"]:
+            if field not in builtin_harmonisation["event"]:
+                # ensure forward-compatibility if we ever remove something from harmonisation
+                continue
+            harmonization["event"][field] = builtin_harmonisation["event"][field]
+            changed = True
+    return changed, configuration, harmonization
+
+
 UPGRADES = OrderedDict([
     ((1, 0, 0, 'dev7'), (v100_dev7_modify_syntax,)),
     ((1, 1, 0), (v110_shadowserver_feednames, v110_deprecations)),
@@ -1004,7 +1032,8 @@ def v340_deprecations(configuration, harmonization, dry_run, **kwargs):
     ((3, 3, 0), ()),
     ((3, 3, 1), ()),
     ((3, 4, 0), (v340_deprecations, )),
-    ((3, 4, 1), ()),
+    ((3, 4, 1), (v341_new_fields, )),
+
 ])
 
 ALWAYS = (harmonization,)

intelmq/tests/bin/initdb.sql

@@ -47,6 +47,11 @@ CREATE TABLE events (
     "misp.attribute_uuid" varchar(36),
     "misp.event_uuid" varchar(36),
     "output" json,
+    "product.full_name" text,
+    "product.name" text,
+    "product.vendor" text,
+    "product.version" text,
+    "product.vulnerabilities" text,
     "protocol.application" varchar(100),
     "protocol.transport" varchar(11),
     "raw" text,

intelmq/tests/lib/test_upgrades.py

@@ -856,6 +856,17 @@ def test_v340_twitter_collector(self):
         self.assertIn('twitter-collector', result[0])
         self.assertEqual(V340_TWITTER_COLLECTOR_IN, result[1])
 
+    def test_v341_new_fields(self):
+        """ Test adding new harmonisation fields """
+        result = upgrades.v341_new_fields({}, {"event": {"old-field": "must stay"}}, False)
+        self.assertTrue(result[0])
+        self.assertIn("old-field", result[2]["event"])
+        self.assertIn("product.full_name", result[2]["event"])
+        self.assertIn("product.name", result[2]["event"])
+        self.assertIn("product.vendor", result[2]["event"])
+        self.assertIn("product.version", result[2]["event"])
+        self.assertIn("product.vulnerabilities", result[2]["event"])
+
 
 for name in upgrades.__all__:
     setattr(TestUpgradeLib, 'test_function_%s' % name,