Merge branch 'main' of https://github.com/chaoss/augur into improve-dockerization

Author: Mbaoma

README.md

@@ -1,4 +1,4 @@
-# Augur NEW Release v0.76.2
+# Augur NEW Release v0.76.4
 
 Augur is primarily a data engineering tool that makes it possible for data scientists to gather open source software community data - less data carpentry for everyone else! 
 The primary way of looking at Augur data is through [8Knot](https://github.com/oss-aspen/8knot), a public instance of 8Knot is available [here](https://metrix.chaoss.io) - this is tied to a public instance of [Augur](https://ai.chaoss.io). 

add.md

@@ -106,7 +106,7 @@ def av_add_user_repo():
             # matches https://gitlab.com/{org}/{repo}/ or http://gitlab.com/{org}/{repo}
             elif Repo.parse_gitlab_repo_url(url)[0]:
 
-                org_name, repo_name = Repo.parse_github_repo_url(url)
+                org_name, repo_name = Repo.parse_gitlab_repo_url(url)
                 repo_git = f"https://gitlab.com/{org_name}/{repo_name}"
 
                 # TODO: gitlab ensure the whole repo git is inserted so it can be found here

augur/api/view/api.py

@@ -1151,7 +1151,7 @@ def extract_needed_gitlab_issue_message_ref_data(message: dict, issue_id: int, r
     return message_ref_dict
 
 
-def extract_needed_gitlab_message_data(comment: dict, platform_id: int, tool_source: str, tool_version: str, data_source: str):
+def extract_needed_gitlab_message_data(comment: dict, platform_id: int, repo_id: int, tool_source: str, tool_version: str, data_source: str):
     """
     Extract specific metadata for a comment from an api response
     and connect it to the relevant platform id.
@@ -1169,6 +1169,7 @@ def extract_needed_gitlab_message_data(comment: dict, platform_id: int, tool_sou
     """
 
     comment_dict = {
+        "repo_id": repo_id,
         "pltfrm_id": platform_id,
         "msg_text": comment['body'],
         "msg_timestamp": comment['created_at'],

augur/application/db/data_parse.py

@@ -26,6 +26,7 @@
 import logging
 import re
 import json
+import urllib.parse
 
 
 from augur.application.db.models.base import Base
@@ -971,7 +972,7 @@ def is_valid_gitlab_repo(gl_session, url: str) -> bool:
             return False, {"status": "Invalid repo URL"}
 
         # Encode namespace and project name for the API request
-        project_identifier = f"{owner}%2F{repo}"
+        project_identifier = urllib.parse.quote(f"{owner}/{repo}", safe='')
         url = REPO_ENDPOINT.format(project_identifier)
 
         attempts = 0
@@ -1030,7 +1031,7 @@ def parse_gitlab_repo_url(url: str) -> tuple:
             Tuple of owner and repo. Or a tuple of None and None if the url is invalid.
         """
 
-        result = re.search(r"https?:\/\/gitlab\.com\/([A-Za-z0-9 \- _]+)\/([A-Za-z0-9 \- _ \.]+)(.git)?\/?$", url)
+        result = re.search(r"https?:\/\/gitlab\.com\/([A-Za-z0-9\-_\/]+)\/([A-Za-z0-9\-_]+)(\.git)?\/?$", url)
 
         if not result:
             return None, None

augur/application/db/models/augur_data.py

@@ -76,7 +76,7 @@ def generate_scorecard(logger, repo_git):
     path = repo_git[8:]
     if path[-4:] == '.git':
         path = path.replace(".git", "")
-    command = '--repo=' + path
+    command = '--local=' + path
 
     #this is path where our scorecard project is located
     path_to_scorecard = os.environ['HOME'] + '/scorecard'
@@ -99,7 +99,7 @@ def generate_scorecard(logger, repo_git):
         logger.info('adding to database...')
         logger.debug(f"output: {required_output}")
 
-        if not required_output['checks']:
+        if not required_output.get('checks'):
             logger.info('No scorecard checks found!')
             return
 

augur/tasks/git/dependency_tasks/core.py

@@ -27,7 +27,7 @@ def process_ossf_dependency_metrics(self, repo_git):
         logger.warning(f'Exception generating scorecard: {e}')
         tracer = ''.join(traceback.format_exception(type(e), e, e.__traceback__))
         logger.warning(f'Full stack trace of OpenSSF scorecard error: {tracer}')
-        raise MetadataException(f"An error occurred while generating the scorecard: {str(e)}")
+        raise MetadataException(e,f"An error occurred while generating the scorecard: {str(e)}")
 
     """
         This try/except block is an attempt to get more information about this occasional error: 

augur/tasks/git/dependency_tasks/tasks.py

@@ -316,7 +316,7 @@ def _collect_and_process_pr_events(self, owner, repo, repo_id, key_auth):
         with engine.connect() as connection:
 
             query = text(f"""
-                select pull_request_id, pr_src_number as gh_pr_number, pr_src_id from pull_requests order by pr_created_at desc; from pull_requests WHERE repo_id={repo_id} order by pr_created_at desc;
+                select pull_request_id, pr_src_number as gh_pr_number, pr_src_id from pull_requests WHERE repo_id={repo_id} order by pr_created_at desc;
             """)
 
             pr_result = connection.execute(query).fetchall()

augur/tasks/github/events.py

@@ -1,6 +1,7 @@
 """Utility functions that are useful for several Github tasks"""
 from typing import Any, List, Tuple
 import logging
+import urllib.parse
 import json
 import httpx
 from augur.tasks.github.util.github_random_key_auth import GithubRandomKeyAuth
@@ -46,6 +47,10 @@ def get_owner_repo(git_url: str) -> Tuple[str, str]:
 
     return owner, repo
 
+def get_gitlab_repo_identifier(owner, repo):
+
+    return urllib.parse.quote(f"{owner}/{repo}", safe='')
+
 
 def parse_json_response(logger: logging.Logger, response: httpx.Response) -> dict:
     # try to get json from response

augur/tasks/github/util/util.py

@@ -7,8 +7,8 @@
 from augur.tasks.init.celery_app import AugurCoreRepoCollectionTask
 from augur.tasks.gitlab.gitlab_api_handler import GitlabApiHandler
 from augur.application.db.data_parse import extract_gitlab_mr_event_data, extract_gitlab_issue_event_data
-from augur.tasks.github.util.util import get_owner_repo
-from augur.application.db.models import Issue, IssueEvent, PullRequest, PullRequestEvent
+from augur.tasks.github.util.util import get_gitlab_repo_identifier
+from augur.application.db.models import Issue, IssueEvent, PullRequest, PullRequestEvent, Repo
 from augur.application.db.lib import bulk_insert_dicts, get_repo_by_repo_git, get_session
 from augur.tasks.gitlab.gitlab_random_key_auth import GitlabRandomKeyAuth
 
@@ -24,7 +24,7 @@ def collect_gitlab_issue_events(repo_git) -> int:
         repo_git: the repo url string
     """
 
-    owner, repo = get_owner_repo(repo_git)
+    owner, repo = Repo.parse_gitlab_repo_url(repo_git)
 
     logger = logging.getLogger(collect_gitlab_issue_events.__name__) 
 
@@ -52,7 +52,7 @@ def collect_gitlab_merge_request_events(repo_git) -> int:
         repo_git: the repo url string
     """
 
-    owner, repo = get_owner_repo(repo_git)
+    owner, repo = Repo.parse_gitlab_repo_url(repo_git)
 
     logger = logging.getLogger(collect_gitlab_issue_events.__name__) 
 
@@ -82,11 +82,13 @@ def retrieve_all_gitlab_event_data(gtype, repo_git, logger, key_auth) -> None:
         key_auth: key auth cache and rotator object 
     """
 
-    owner, repo = get_owner_repo(repo_git)
+    owner, repo = Repo.parse_gitlab_repo_url(repo_git)
+
+    repo_identifier = get_gitlab_repo_identifier(owner, repo)
 
     logger.info(f"Collecting gitlab issue events for {owner}/{repo}")
 
-    url = f"https://gitlab.com/api/v4/projects/{owner}%2f{repo}/events?target_type={gtype}"
+    url = f"https://gitlab.com/api/v4/projects/{repo_identifier}/events?target_type={gtype}"
     events = GitlabApiHandler(key_auth, logger)
 
     all_data = []