Update rules with required phrases automatically

docs/source/how-to-guides/add_new_license_detection_rule.rst

@@ -73,7 +73,7 @@ More (advanced) rules options:
   be present in the result license detections. These just have the license text and a
   `is_false_positive` flag set to True.
 
-- you can specify key phrases by surrounding one or more words between the `{{`
+- you can specify required phrases by surrounding one or more words between the `{{`
   and `}}` tags. Key phrases are words that **must** be matched/present in order
   for a RULE to be considered a match.
 

etc/scripts/licenses/buildrules.py

@@ -16,6 +16,7 @@
 from licensedcode import models
 from licensedcode import match_hash
 from licensedcode import frontmatter
+from licensedcode.models import get_rule_id_for_text
 from license_expression import Licensing
 
 """
@@ -130,23 +131,6 @@ def load_data(location="00-new-licenses.txt"):
     return rules
 
 
-def rule_exists(text):
-    """
-    Return the matched rule identifier if the text is an existing rule matched
-    exactly, False otherwise.
-    """
-    idx = cache.get_index()
-
-    matches = idx.match(query_string=text)
-    if not matches:
-        return False
-    if len(matches) > 1:
-        return False
-    match = matches[0]
-    if match.matcher == match_hash.MATCH_HASH and match.score() == 100:
-        return match.rule.identifier
-
-
 def all_rule_by_tokens():
     """
     Return a mapping of {tuples of tokens: rule id}, with one item for each
@@ -347,7 +331,7 @@ def cli(licenses_file, dump_to_file_on_errors=False):
 
         text = rule.text
 
-        existing_rule = rule_exists(text)
+        existing_rule = get_rule_id_for_text(text)
         skinny_text = " ".join(text[:80].split()).replace("{", " ").replace("}", " ")
 
         existing_msg = (

etc/scripts/licenses/report_license_rules.py

@@ -62,6 +62,8 @@
     "is_license_reference",
     "is_license_intro",
     "is_license_clue",
+    "is_required_phrase",
+    "skip_for_required_phrase_generation",
     "is_deprecated",
     "has_unknown",
     "only_known_words",

respective.txt

@@ -0,0 +1,3 @@
+ This software is (C) by the respective authors, and licensed under the GPL
+ License.
+

setup-mini.cfg

@@ -158,6 +158,7 @@ console_scripts =
     scancode-reindex-licenses = licensedcode.reindex:reindex_licenses
     scancode-license-data = licensedcode.license_db:dump_scancode_license_data
     regen-package-docs = packagedcode.regen_package_docs:regen_package_docs
+    add-required-phrases = licensedcode.required_phrases:add_required_phrases
 
 # These are configurations for ScanCode plugins as setuptools entry points.
 # Each plugin entry hast this form:

setup.cfg

@@ -158,6 +158,8 @@ console_scripts =
     scancode-reindex-licenses = licensedcode.reindex:reindex_licenses
     scancode-license-data = licensedcode.license_db:dump_scancode_license_data
     regen-package-docs = packagedcode.regen_package_docs:regen_package_docs
+    add-required-phrases = licensedcode.required_phrases:add_required_phrases
+	gen-new-required-phrases-rules = licensedcode.required_phrases:gen_required_phrases_rules
 
 # These are configurations for ScanCode plugins as setuptools entry points.
 # Each plugin entry hast this form:

src/licensedcode/cache.py

@@ -9,6 +9,7 @@
 
 import os
 import pickle
+
 from shutil import rmtree
 
 from commoncode.fileutils import create_dir
@@ -185,10 +186,7 @@
                     additional_license_plugins=plugin_directories,
                 )
 
-                # save the cache as pickle new tree checksum
-                with open(cache_file, 'wb') as fn:
-                    pickle.dump(license_cache, fn, protocol=PICKLE_PROTOCOL)
-
+                license_cache.dump(cache_file)
                 return license_cache
 
         except lockfile.LockTimeout:
@@ -201,6 +199,13 @@
         if cache.additional_license_directory or cache.additional_license_plugins:
             return True
 
+    def dump(self, cache_file):
+        """
+        Dump this license cache on disk at ``cache_file``.
+        """
+        with open(cache_file, 'wb') as fn:
+            pickle.dump(self, fn, protocol=PICKLE_PROTOCOL)
+
 
 def build_index(
     licenses_db=None,
@@ -235,9 +240,12 @@
     if not licenses_db:
         # combine the licenses in these additional directories with the licenses in the original DB
         additional_license_dirs = get_license_dirs(additional_dirs=additional_directories)
-        combined_license_directories = [licenses_data_dir] + additional_license_dirs
         # generate a single combined license db with all licenses
-        licenses_db = load_licenses_from_multiple_dirs(license_dirs=combined_license_directories)
+        licenses_db = load_licenses_from_multiple_dirs(
+            builtin_license_data_dir=licenses_data_dir,
+            additional_license_data_dirs=additional_license_dirs,
+            with_deprecated=False,
+        )
 
     # if we have additional directories, extract the rules from them
     additional_rule_dirs = get_rule_dirs(additional_dirs=additional_directories)
@@ -393,7 +401,7 @@
     Return a LicenseCache either rebuilt, cached or loaded from disk.
 
     If ``index_all_languages`` is True, include texts in all languages when
-    building the license index. Otherwise, only include the English license \
+    building the license index. Otherwise, only include the English license
     texts and rules (the default)
     """
     return populate_cache(
@@ -531,7 +539,7 @@
         if not type(key) == str:
             msg = f"Invalid license key: {key} of type {type(key)}, license key should be a string"
             messages.append(msg)
-    
+
         lic = license_db.get(key, None)
         if not lic:
             licenses = load_licenses(with_deprecated=True)

src/licensedcode/data/licenses/any-osi-perl-modules.LICENSE

@@ -11,6 +11,7 @@ other_urls:
     - https://metacpan.org/pod/Net::MQTT::Simple#LICENSE
 ignorable_urls:
     - http://www.opensource.org/licenses/alphabetical
+minimum_coverage: 90
 ---
 
 This software may be redistributed under the terms of the GPL, LGPL,
@@ -23,4 +24,4 @@ When using a packaged version, please refer to the package metadata to see
 under which license terms it was distributed. Alternatively, a distributor
 may choose to replace the LICENSE section of the documentation and/or
 include a LICENSE file to reflect the license(s) they chose to redistribute
-under.
+under.

src/licensedcode/data/licenses/dco-1.0.LICENSE

@@ -0,0 +1,39 @@
+---
+key: dco-1.0
+short_name: DCO 1.0
+name: Developer Certificate of Origin 1.0
+category: CLA
+owner: Linux Foundation
+homepage_url: https://developercertificate.org/
+spdx_license_key: LicenseRef-scancode-dco-1.0
+text_urls:
+    - https://developercertificate.org/
+other_urls:
+    - https://github.com/nexB/scancode-toolkit/issues/3038#issuecomment-1317511139
+minimum_coverage: 90
+---
+
+Developer's Certificate of Origin 1.0
+
+By making a contribution to this project, I certify that:
+
+1. The contribution was created in whole or in part by me and I
+   have the right to submit it under the open source license
+   indicated in the file LICENSE; or
+
+2. The contribution is based upon previous work that, to the best
+   of my knowledge, is covered under an appropriate open source
+   license and I have the right under that license to submit that
+   work with modifications, whether created in whole or in part
+   by me, under the same open source license (unless I am
+   permitted to submit under a different license), as indicated
+   in the file LICENSE; or
+
+3. The contribution was provided directly to me by some other
+   person who certified (1), (2) or (3) and I have not modified
+   it.
+
+4. I understand and agree that this project and the contribution
+   are public and that a record of the contribution (including all
+   personal information I submit with it, including my sign-off) is
+   maintained indefinitely and may be redistributed consistent

src/licensedcode/data/licenses/samba-dco-1.0.LICENSE

@@ -0,0 +1,51 @@
+---
+key: samba-dco-1.0
+short_name: Samba Developer's Declaration, Version 1.0
+name: Samba Developer's Declaration, Version 1.0
+category: CLA
+owner: Samba
+homepage_url: https://github.com/samba-team/samba/blob/master/README.contributing
+spdx_license_key: LicenseRef-scancode-samba-dco-1.0
+text_urls:
+    - https://github.com/samba-team/samba/blob/master/README.contributing
+minimum_coverage: 80
+ignorable_urls:
+    - http://www.gnu.org/licenses/gpl-3.0.html
+    - http://www.gnu.org/licenses/lgpl-3.0.html
+---
+
+Samba Developer's Declaration, Version 1.0
+
+By making a contribution to this project, I certify that:
+
+(a) The contribution was created in whole or in part by me and I
+    have the right to submit it under the appropriate
+    version of the GNU General Public License; or
+
+(b) The contribution is based upon previous work that, to the best
+    of my knowledge, is covered under an appropriate open source
+    license and I have the right under that license to submit that
+    work with modifications, whether created in whole or in part
+    by me, under the GNU General Public License, in the
+    appropriate version; or
+
+(c) The contribution was provided directly to me by some other
+    person who certified (a) or (b) and I have not modified
+    it.
+
+(d) I understand and agree that this project and the
+    contribution are public and that a record of the
+    contribution (including all metadata and personal
+    information I submit with it, including my sign-off) is
+    maintained indefinitely and may be redistributed
+    consistent with the Samba Team's policies and the
+    requirements of the GNU GPL where they are relevant.
+
+(e) I am granting this work to this project under the terms of both
+    the GNU General Public License and the GNU Lesser General Public
+    License as published by the Free Software Foundation; either version
+    3 of these Licenses, or (at the option of the project) any later
+    version.
+
+    http://www.gnu.org/licenses/gpl-3.0.html
+    http://www.gnu.org/licenses/lgpl-3.0.html

src/licensedcode/data/rules/agpl-1.0-plus_62.RULE

@@ -7,4 +7,4 @@ minimum_coverage: 100
 notes: Rule based on an SPDX license name and/or ID
 ---
 
-AGPL-1.0+ Affero General Public License v1.0 or later
+{{AGPL-1.0+}} {{Affero General Public License v1.0 or later}}

src/licensedcode/data/rules/agpl-1.0-plus_63.RULE

@@ -1,6 +1,7 @@
 ---
 license_expression: agpl-1.0-plus
 is_license_reference: yes
+is_required_phrase: yes
 is_continuous: yes
 relevance: 100
 minimum_coverage: 100

src/licensedcode/data/rules/agpl-1.0-plus_64.RULE

@@ -7,4 +7,4 @@ minimum_coverage: 100
 notes: Rule based on an SPDX license name and/or ID
 ---
 
-Affero General Public License v1.0 or later AGPL-1.0+
+{{Affero General Public License v1.0 or later}} {{AGPL-1.0+}}

src/licensedcode/data/rules/agpl-1.0-plus_65.RULE

@@ -1,6 +1,7 @@
 ---
 license_expression: agpl-1.0-plus
 is_license_tag: yes
+is_required_phrase: yes
 is_continuous: yes
 relevance: 100
 minimum_coverage: 100

src/licensedcode/data/rules/agpl-1.0-plus_66.RULE

@@ -7,4 +7,4 @@ minimum_coverage: 100
 notes: Rule based on an SPDX license name and/or ID
 ---
 
-licenseid : AGPL-1.0+
+licenseid : {{AGPL-1.0+}}

src/licensedcode/data/rules/agpl-1.0-plus_67.RULE

@@ -7,4 +7,4 @@ minimum_coverage: 100
 notes: Rule based on an SPDX license name and/or ID
 ---
 
-name : Affero General Public License v1.0 or later
+name : {{Affero General Public License v1.0 or later}}

src/licensedcode/data/rules/agpl-1.0-plus_68.RULE

@@ -7,4 +7,4 @@ minimum_coverage: 100
 notes: Rule based on an SPDX license identifier and name
 ---
 
-AGPL-1.0-or-later Affero General Public License v1.0 or later
+{{AGPL-1.0-or-later}} {{Affero General Public License v1.0 or later}}

src/licensedcode/data/rules/agpl-1.0-plus_69.RULE

@@ -7,4 +7,4 @@ minimum_coverage: 100
 notes: Rule based on an SPDX license identifier and name
 ---
 
-Affero General Public License v1.0 or later AGPL-1.0-or-later
+{{Affero General Public License v1.0 or later}} {{AGPL-1.0-or-later}}

src/licensedcode/data/rules/agpl-1.0-plus_70.RULE

@@ -7,4 +7,4 @@ minimum_coverage: 100
 notes: Rule based on an SPDX license identifier and name
 ---
 
-license: AGPL-1.0-or-later
+license: {{AGPL-1.0-or-later}}

src/licensedcode/data/rules/agpl-1.0-plus_71.RULE

@@ -7,4 +7,4 @@ minimum_coverage: 100
 notes: Rule based on an SPDX license identifier and name
 ---
 
-license: Affero General Public License v1.0 or later
+license: {{Affero General Public License v1.0 or later}}

src/licensedcode/data/rules/agpl-1.0-plus_72.RULE

@@ -7,4 +7,4 @@ minimum_coverage: 100
 notes: Rule based on an SPDX license identifier and name
 ---
 
-licenseId: AGPL-1.0-or-later
+licenseId: {{AGPL-1.0-or-later}}

src/licensedcode/data/rules/agpl-1.0-plus_74.RULE

@@ -4,4 +4,4 @@ is_license_reference: yes
 relevance: 100
 ---
 
-licenses.nuget.org/AGPL-1.0-or-later
+licenses.nuget.org/{{AGPL-1.0-or-later}}

src/licensedcode/data/rules/agpl-1.0_5.RULE

@@ -8,5 +8,5 @@ referenced_filenames:
 // The source code packaged with this file is Free Software, 
 // It's licensed under the AFFERO GENERAL PUBLIC LICENSE unless stated otherwise.
 // You can get copies of the licenses here:
-// affero.org: Affero General Public License version 1 (AGPLv1)
+// affero.org: {{Affero General Public License version 1}} (AGPLv1)
 // AFFERO GENERAL PUBLIC LICENSE is also included in the file called "COPYING".

src/licensedcode/data/rules/agpl-3.0-plus_100.RULE

@@ -9,4 +9,4 @@ ignorable_urls:
 ---
 
 Licensing
-This project is free and open-source software and is distributed under the terms of the GNU Affero General Public License v3 or any later version. It is subject to the license terms in the LICENSE file found in the top-level directory of this distribution and at https://www.gnu.org/licenses/agpl-3.0.md. By submitting a contribution to this project you agree and acknowledge that your contribution may be used under the terms of the GNU Affero General Public License v3 or any later version and you are authorized to submit the contribution on behalf of the copyright holder. No part of this project, including this file, may be copied, modified, propagated, or distributed except according to the terms contained in the LICENSE file.
+This project is free and open-source software and is distributed under the terms of the {{GNU Affero General Public License}} v3 or any later version. It is subject to the license terms in the LICENSE file found in the top-level directory of this distribution and at https://www.gnu.org/licenses/agpl-3.0.md. By submitting a contribution to this project you agree and acknowledge that your contribution may be used under the terms of the {{GNU Affero General Public License}} v3 or any later version and you are authorized to submit the contribution on behalf of the copyright holder. No part of this project, including this file, may be copied, modified, propagated, or distributed except according to the terms contained in the LICENSE file.

src/licensedcode/data/rules/agpl-3.0-plus_106.RULE

@@ -4,4 +4,4 @@ is_license_notice: yes
 relevance: 100
 ---
 
-This program is licensed under the GNU Affero General Public License.
+This program is licensed under the {{GNU Affero General Public License}}.

src/licensedcode/data/rules/agpl-3.0-plus_107.RULE

@@ -4,4 +4,4 @@ is_license_notice: yes
 relevance: 100
 ---
 
-program is licensed under the GNU Affero General Public License.
+program is licensed under the {{GNU Affero General Public License}}.