From 5e4bef85985c1676e54f92cfb0372f7c8fd9924d Mon Sep 17 00:00:00 2001 From: Alan Tse Date: Tue, 21 Jun 2022 14:23:59 -0700 Subject: [PATCH 01/13] Pass date to template --- .../generate_html_notices_from_json.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/examples/generate_html_notices_report_from_json/generate_html_notices_from_json.py b/examples/generate_html_notices_report_from_json/generate_html_notices_from_json.py index 6db90c0c..e5d7881f 100644 --- a/examples/generate_html_notices_report_from_json/generate_html_notices_from_json.py +++ b/examples/generate_html_notices_report_from_json/generate_html_notices_from_json.py @@ -2,12 +2,14 @@ import os import json import argparse +import datetime parser = argparse.ArgumentParser("A program to transform a JSON formatted representation of a Black Duck notices file into a standalone HTML document") parser.add_argument("json_file", help="The input JSON file to be converted") parser.add_argument("output_file_html", help="The file to output the results to") args = parser.parse_args() +date = datetime.date.today() templates_dir = os.path.dirname(os.path.abspath(__file__)) env = Environment(loader=FileSystemLoader(templates_dir)) @@ -21,4 +23,6 @@ fh.write(template.render(componentLicenses=fileContent['componentLicenses'], licenseTexts=fileContent['licenseTexts'], componentCopyrightTexts=fileContent['componentCopyrightTexts'], - projectVersion=fileContent['projectVersion'])) \ No newline at end of file + projectVersion=fileContent['projectVersion'], + date=date + )) \ No newline at end of file From 933e5d392185e3dda6e6cac83615181607ec2952 Mon Sep 17 00:00:00 2001 From: Alan Tse Date: Tue, 21 Jun 2022 16:09:16 -0700 Subject: [PATCH 02/13] Refactor collapse into function --- .../notices-template.html | 34 +++++-------------- 1 file changed, 8 insertions(+), 26 deletions(-) diff --git a/examples/generate_html_notices_report_from_json/notices-template.html b/examples/generate_html_notices_report_from_json/notices-template.html index 64496679..53507137 100644 --- a/examples/generate_html_notices_report_from_json/notices-template.html +++ b/examples/generate_html_notices_report_from_json/notices-template.html @@ -83,10 +83,11 @@

{{ row.name }}

From 156095d9ca5f938a8a0fa4a0988d0d827b7f489f Mon Sep 17 00:00:00 2001 From: Alan Tse Date: Tue, 21 Jun 2022 17:20:11 -0700 Subject: [PATCH 03/13] Add links to subsections --- .../generate_html_notices_from_json.py | 2 ++ .../notices-template.html | 24 +++++++++++++++---- .../requirements.txt | 1 + 3 files changed, 22 insertions(+), 5 deletions(-) diff --git a/examples/generate_html_notices_report_from_json/generate_html_notices_from_json.py b/examples/generate_html_notices_report_from_json/generate_html_notices_from_json.py index e5d7881f..bc3f81c3 100644 --- a/examples/generate_html_notices_report_from_json/generate_html_notices_from_json.py +++ b/examples/generate_html_notices_report_from_json/generate_html_notices_from_json.py @@ -1,4 +1,5 @@ from jinja2 import Environment, FileSystemLoader +from slugify import slugify import os import json import argparse @@ -13,6 +14,7 @@ templates_dir = os.path.dirname(os.path.abspath(__file__)) env = Environment(loader=FileSystemLoader(templates_dir)) +env.filters['slugify'] = slugify template = env.get_template('notices-template.html') with open(args.output_file_html, 'w+') as fh: diff --git a/examples/generate_html_notices_report_from_json/notices-template.html b/examples/generate_html_notices_report_from_json/notices-template.html index 53507137..256e0866 100644 --- a/examples/generate_html_notices_report_from_json/notices-template.html +++ b/examples/generate_html_notices_report_from_json/notices-template.html @@ -20,8 +20,8 @@

Components

- - + + @@ -29,18 +29,29 @@

Components

@@ -53,7 +64,7 @@

Components

Copyright Data

ComponentLicenseComponent (click for copyrights)License (click for license)
- {{ row.component.projectName }} + {% set componentCopyrightFound = (componentCopyrightTexts|length > 0 and + componentCopyrightTexts | selectattr("componentVersionSummary.projectName", "equalto", row.component.projectName) | selectattr("componentVersionSummary.versionName", "equalto", row.component.versionName) | list | length > 0 ) %} + {% if componentCopyrightFound %} + + {% endif %} + + {{ row.component.projectName }} {% if row.component.versionName %} {{ row.component.versionName }} {% endif %} + {% if componentCopyrightFound %} + + {% endif %}
{% if row.licenses|length > 0 %} + {{ row.licenses[0]['name'] }} + {% endif %}
From 455152e6dfe15014ab15cd72364e9053b02240c8 Mon Sep 17 00:00:00 2001 From: Alan Tse Date: Tue, 21 Jun 2022 17:21:27 -0700 Subject: [PATCH 05/13] Add head section --- examples/generate_html_notices_report_from_json/head.html | 3 +++ .../notices-template.html | 1 + 2 files changed, 4 insertions(+) create mode 100644 examples/generate_html_notices_report_from_json/head.html diff --git a/examples/generate_html_notices_report_from_json/head.html b/examples/generate_html_notices_report_from_json/head.html new file mode 100644 index 00000000..be553f8c --- /dev/null +++ b/examples/generate_html_notices_report_from_json/head.html @@ -0,0 +1,3 @@ + + {{ projectVersion.projectName }} — {{ projectVersion.versionName }} — Notices File + \ No newline at end of file diff --git a/examples/generate_html_notices_report_from_json/notices-template.html b/examples/generate_html_notices_report_from_json/notices-template.html index f6334c95..0be46ab7 100644 --- a/examples/generate_html_notices_report_from_json/notices-template.html +++ b/examples/generate_html_notices_report_from_json/notices-template.html @@ -1,5 +1,6 @@ +{% include 'head.html' %} {% include 'intro.html' %} From 43fbe8377d2218644f0ac23aaefad5c14a1149d6 Mon Sep 17 00:00:00 2001 From: Alan Tse Date: Wed, 22 Jun 2022 11:46:34 -0700 Subject: [PATCH 06/13] Ignores restconfig.json --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 1c20e402..ef8b881c 100644 --- a/.gitignore +++ b/.gitignore @@ -103,3 +103,6 @@ venv.bak/ # mypy .mypy_cache/ + +#restconfig +.restconfig.json \ No newline at end of file From ea8a79fd9ce24d90a574caec658928306ee921cd Mon Sep 17 00:00:00 2001 From: Alan Tse Date: Wed, 29 Jun 2022 16:55:45 -0700 Subject: [PATCH 07/13] Fix encoding bug in Windows Encode the template string in utf-8 prior to writing to the file handler --- .../generate_html_notices_from_json.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/generate_html_notices_report_from_json/generate_html_notices_from_json.py b/examples/generate_html_notices_report_from_json/generate_html_notices_from_json.py index bc3f81c3..d70fbba0 100644 --- a/examples/generate_html_notices_report_from_json/generate_html_notices_from_json.py +++ b/examples/generate_html_notices_report_from_json/generate_html_notices_from_json.py @@ -17,7 +17,7 @@ env.filters['slugify'] = slugify template = env.get_template('notices-template.html') -with open(args.output_file_html, 'w+') as fh: +with open(args.output_file_html, 'wb+') as fh: with open(args.json_file, 'r') as lj: data = json.load(lj) fileContent = data['reportContent'][0]['fileContent'] @@ -27,4 +27,4 @@ componentCopyrightTexts=fileContent['componentCopyrightTexts'], projectVersion=fileContent['projectVersion'], date=date - )) \ No newline at end of file + ).encode("utf-8")) From 242495939a776360ed56a71a1d761b3b401021f9 Mon Sep 17 00:00:00 2001 From: Alan Tse Date: Thu, 23 Jun 2022 17:26:40 -0700 Subject: [PATCH 08/13] Process and filter copyrightTexts --- .../generate_html_notices_from_json.py | 36 ++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/examples/generate_html_notices_report_from_json/generate_html_notices_from_json.py b/examples/generate_html_notices_report_from_json/generate_html_notices_from_json.py index d70fbba0..ad63726f 100644 --- a/examples/generate_html_notices_report_from_json/generate_html_notices_from_json.py +++ b/examples/generate_html_notices_report_from_json/generate_html_notices_from_json.py @@ -4,6 +4,7 @@ import json import argparse import datetime +import re parser = argparse.ArgumentParser("A program to transform a JSON formatted representation of a Black Duck notices file into a standalone HTML document") parser.add_argument("json_file", help="The input JSON file to be converted") @@ -16,6 +17,39 @@ env = Environment(loader=FileSystemLoader(templates_dir)) env.filters['slugify'] = slugify template = env.get_template('notices-template.html') +copyrightFilters = [ + re.compile(r".*[0-9]{4,}.*"), # at least four numbers for years. years with 2 digits tend to follow 4 (e.g., 1991, 92, 93) + # an email address https://stackoverflow.com/a/201378 + re.compile(r'''.*(?:[a-z0-9!#$%&'*+/=?^_`{|}~-]+(?:\.[a-z0-9!#$%&'*+/=?^_`{|}~-]+)*|"(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21\x23-\x5b\x5d-\x7f]|\\[\x01-\x09\x0b\x0c\x0e-\x7f])*")@(?:(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?|\[(?:(?:(2(5[0-5]|[0-4][0-9])|1[0-9][0-9]|[1-9]?[0-9]))\.){3}(?:(2(5[0-5]|[0-4][0-9])|1[0-9][0-9]|[1-9]?[0-9])|[a-z0-9-]*[a-z0-9]:(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21-\x5a\x53-\x7f]|\\[\x01-\x09\x0b\x0c\x0e-\x7f])+)\]).*'''), + re.compile(r"copyright", re.IGNORECASE), # an explicit copyright statement +] + +def processCopyrightText(copyrightTexts: dict, filters:list=[]) -> dict: + """Sort, filter, and remove duplicates from copyright texts from componentCopyrightTexts. + + The algorithm for Copyright Texts in Black Duck are very conservative and result in a lot of potentially false positives. This function will attempt to remove false positives and order the list. + + Args: + copyrightTexts (dict): Dictionary with list of copyrights in key copyrightTexts + filters (list, optional): List of regex pattern filters. Matching any filters will keep the item. Defaults to []. + + Returns: + dict: A processed version of copyrightTexts. + """ + from collections import OrderedDict + if copyrightTexts: + for component in copyrightTexts: + copyrightlist = component.get("copyrightTexts") + if filters: + filteredlist = [] + for filterRegex in filters: + filteredlist += list(filter(lambda x: filterRegex.match(x), copyrightlist)) + filteredlist + copyrightlist = map(lambda x: x.strip(),filteredlist) + if copyrightlist: + component["copyrightTexts"] = list(OrderedDict.fromkeys(copyrightlist)) + component["copyrightTexts"].sort() + return copyrightTexts with open(args.output_file_html, 'wb+') as fh: with open(args.json_file, 'r') as lj: @@ -24,7 +58,7 @@ fh.write(template.render(componentLicenses=fileContent['componentLicenses'], licenseTexts=fileContent['licenseTexts'], - componentCopyrightTexts=fileContent['componentCopyrightTexts'], + componentCopyrightTexts=processCopyrightText(fileContent['componentCopyrightTexts'], copyrightFilters), projectVersion=fileContent['projectVersion'], date=date ).encode("utf-8")) From 1ea7deddfd09f940a451611fd618531575989106 Mon Sep 17 00:00:00 2001 From: Alan Tse Date: Thu, 23 Jun 2022 18:32:16 -0700 Subject: [PATCH 09/13] Add component, copyright, and license counts --- .../notices-template.html | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/generate_html_notices_report_from_json/notices-template.html b/examples/generate_html_notices_report_from_json/notices-template.html index 0be46ab7..52de8986 100644 --- a/examples/generate_html_notices_report_from_json/notices-template.html +++ b/examples/generate_html_notices_report_from_json/notices-template.html @@ -4,7 +4,7 @@ {% include 'intro.html' %} -

Components

+

Components ({{ componentLicenses | length}})

@@ -49,7 +49,7 @@

Components

{% if componentCopyrightTexts|length > 0 %} -

Copyright Data

+

Copyright Data ({{ componentCopyrightTexts|length }})