Merge branch 'dev' into jumpstart

Signed-off-by: Ulincsys <[email protected]>

Author: Ulincsys

README.md

@@ -1,4 +1,4 @@
-# Augur NEW Release v0.76.0
+# Augur NEW Release v0.76.1
 
 Augur is primarily a data engineering tool that makes it possible for data scientists to gather open source software community data. Less data carpentry for everyone else! 
 The primary way of looking at Augur data is through [8Knot](https://github.com/oss-aspen/8knot) ... A public instance of 8Knot is available at https://metrix.chaoss.io ... That is tied to a public instance of Augur at https://ai.chaoss.io 
@@ -10,7 +10,7 @@ The primary way of looking at Augur data is through [8Knot](https://github.com/o
 ## NEW RELEASE ALERT!
 ### [If you want to jump right in, updated docker build/compose and bare metal installation instructions are available here](docs/new-install.md)
 
-Augur is now releasing a dramatically improved new version to the main branch. It is also available here: https://github.com/chaoss/augur/releases/tag/v0.76.0  
+Augur is now releasing a dramatically improved new version to the main branch. It is also available here: https://github.com/chaoss/augur/releases/tag/v0.76.1  
 
 - The `main` branch is a stable version of our new architecture, which features:
   - Dramatic improvement in the speed of large scale data collection (100,000+ repos). All data is obtained for 100k+ repos within 2 weeks.

augur/api/routes/pull_request_reports.py

@@ -21,6 +21,12 @@
 from bokeh.models.glyphs import Rect
 from bokeh.transform import dodge, factor_cmap, transform
 
+# from selenium.webdriver import Firefox, FirefoxOptions
+# options = FirefoxOptions()
+# options.headless = True
+# webdriver = Firefox(options=options)
+#export_png(item, path, webdriver=webdriver)
+
 warnings.filterwarnings('ignore')
 
 from augur.api.routes import AUGUR_API_VERSION
@@ -604,6 +610,7 @@ def average_commits_per_PR():
         # opts = FirefoxOptions()
     # opts.add_argument("--headless")
     # driver = webdriver.Firefox(firefox_options=opts)
+    # filename = export_png(grid, timeout=180, webdriver=webdriver)
     filename = export_png(grid, timeout=180)
 
     return send_file(filename)

augur/application/cli/backend.py

@@ -194,7 +194,7 @@ def determine_worker_processes(ratio,maximum):
         sleep_time += 6
 
         #60% of estimate, Maximum value of 45 : Reduced because it can be lower
-        core_num_processes = determine_worker_processes(.40, 50)
+        core_num_processes = determine_worker_processes(.40, 90)
         logger.info(f"Starting core worker processes with concurrency={core_num_processes}")
         core_worker = f"celery -A augur.tasks.init.celery_app.celery_app worker -l info --concurrency={core_num_processes} -n core:{uuid.uuid4().hex}@%h"
         process_list.append(subprocess.Popen(core_worker.split(" ")))

augur/application/cli/collection.py

@@ -126,7 +126,7 @@ def determine_worker_processes(ratio,maximum):
     sleep_time += 6
 
     #60% of estimate, Maximum value of 45: Reduced because not needed
-    core_num_processes = determine_worker_processes(.40, 50)
+    core_num_processes = determine_worker_processes(.40, 90)
     logger.info(f"Starting core worker processes with concurrency={core_num_processes}")
     core_worker = f"celery -A augur.tasks.init.celery_app.celery_app worker -l info --concurrency={core_num_processes} -n core:{uuid.uuid4().hex}@%h"
     process_list.append(subprocess.Popen(core_worker.split(" ")))

augur/application/cli/tasks.py

@@ -36,8 +36,8 @@ def start():
     secondary_worker_process = None
 
     scheduling_worker = f"celery -A augur.tasks.init.celery_app.celery_app worker -l info --concurrency=1 -n scheduling:{uuid.uuid4().hex}@%h -Q scheduling"
-    core_worker = f"celery -A augur.tasks.init.celery_app.celery_app worker -l info --concurrency=50 -n core:{uuid.uuid4().hex}@%h"
-    secondary_worker = f"celery -A augur.tasks.init.celery_app.celery_app worker -l info --concurrency=50 -n secondary:{uuid.uuid4().hex}@%h -Q secondary"
+    core_worker = f"celery -A augur.tasks.init.celery_app.celery_app worker -l info --concurrency=90 -n core:{uuid.uuid4().hex}@%h"
+    secondary_worker = f"celery -A augur.tasks.init.celery_app.celery_app worker -l info --concurrency=20 -n secondary:{uuid.uuid4().hex}@%h -Q secondary"
 
     scheduling_worker_process = subprocess.Popen(scheduling_worker.split(" "))
     core_worker_process = subprocess.Popen(core_worker.split(" "))

augur/application/db/models/__init__.py

@@ -63,6 +63,7 @@
     PullRequestTeam,
     PullRequestRepo,
     PullRequestReviewMessageRef,
+    CommitMessage,
     RepoClone,
 )
 

augur/application/db/models/augur_data.py

@@ -1350,6 +1350,36 @@ class Commit(Base):
     repo = relationship("Repo", back_populates="commits")
     message_ref = relationship("CommitCommentRef", back_populates="cmt")
 
+class CommitMessage(Base):
+    __tablename__ = "commit_messages"
+    __table_args__ = ( UniqueConstraint("repo_id","cmt_hash", name="commit-message-insert-unique"),
+        { 
+            "schema": "augur_data",
+            "comment": "This table holds commit messages",
+        }
+    )
+
+    cmt_msg_id = Column(
+        BigInteger,
+        primary_key=True,
+        server_default=text("nextval('augur_data.commits_cmt_id_seq'::regclass)"),
+    )
+
+    repo_id = Column(
+        ForeignKey("augur_data.repo.repo_id", ondelete="RESTRICT", onupdate="CASCADE"),
+        nullable=False,
+    )
+
+    cmt_msg = Column(String, nullable=False)
+
+    cmt_hash = Column(String(80), nullable=False)
+
+    tool_source = Column(String)
+    tool_version = Column(String)
+    data_source = Column(String)
+    data_collection_date = Column(
+        TIMESTAMP(precision=0), server_default=text("CURRENT_TIMESTAMP")
+    )
 
 class Issue(Base):
     __tablename__ = "issues"

augur/application/db/models/augur_operations.py

@@ -30,39 +30,38 @@ def retrieve_owner_repos(session, owner: str) -> List[str]:
     Returns
         List of valid repo urls or empty list if invalid org
     """
-    from augur.tasks.github.util.github_paginator import GithubPaginator, retrieve_dict_from_endpoint
+    from augur.tasks.github.util.github_data_access import GithubDataAccess, UrlNotFoundException
 
     OWNER_INFO_ENDPOINT = f"https://api.github.com/users/{owner}"
     ORG_REPOS_ENDPOINT = f"https://api.github.com/orgs/{owner}/repos?per_page=100"
     USER_REPOS_ENDPOINT = f"https://api.github.com/users/{owner}/repos?per_page=100"
 
+    github_data_access = GithubDataAccess(session.oauths, logger)
+
     if not session.oauths.list_of_keys:
         return None, {"status": "No valid github api keys to retrieve data with"}
 
     # determine whether the owner is a user or an organization
-    data, _ = retrieve_dict_from_endpoint(logger, session.oauths, OWNER_INFO_ENDPOINT)
-    if not data:
+    try: 
+        data = github_data_access.get_resource(OWNER_INFO_ENDPOINT)
+    except UrlNotFoundException as e:
+        logger.error("Owner not found on github")
         return None, {"status": "Invalid owner"}
+    except Exception as e:
+        logger.error(f"Failed to get owner data from github. Exception: {e}")
+        return None, {"status": "Failed to get owner data from github"}
 
     owner_type = data["type"]
 
-
     if owner_type == "User":
         url = USER_REPOS_ENDPOINT
     elif owner_type == "Organization":
         url = ORG_REPOS_ENDPOINT
     else:
         return None, {"status": f"Invalid owner type: {owner_type}"}
 
-
     # collect repo urls for the given owner
-    repos = []
-    for page_data, _ in GithubPaginator(url, session.oauths, logger).iter_pages():
-
-        if page_data is None:
-            break
-
-        repos.extend(page_data)
+    repos = list(github_data_access.paginate_resource(url))
 
     repo_urls = [repo["html_url"] for repo in repos]
 

augur/application/schema/alembic/versions/29_add_commit_message_table.py

@@ -0,0 +1,42 @@
+"""Add commit message table
+
+Revision ID: 29
+Revises: 28
+Create Date: 2024-07-25 12:02:57.185867
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = '29'
+down_revision = '28'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('commit_messages',
+    sa.Column('cmt_msg_id', sa.BigInteger(), server_default=sa.text("nextval('augur_data.commits_cmt_id_seq'::regclass)"), nullable=False),
+    sa.Column('repo_id', sa.BigInteger(), nullable=False),
+    sa.Column('cmt_msg', sa.String(), nullable=False),
+    sa.Column('cmt_hash', sa.String(length=80), nullable=False),
+    sa.Column('tool_source', sa.String(), nullable=True),
+    sa.Column('tool_version', sa.String(), nullable=True),
+    sa.Column('data_source', sa.String(), nullable=True),
+    sa.Column('data_collection_date', postgresql.TIMESTAMP(precision=0), server_default=sa.text('CURRENT_TIMESTAMP'), nullable=True),
+    sa.ForeignKeyConstraint(['repo_id'], ['augur_data.repo.repo_id'], onupdate='CASCADE', ondelete='RESTRICT'),
+    sa.PrimaryKeyConstraint('cmt_msg_id'),
+    sa.UniqueConstraint('repo_id', 'cmt_hash', name='commit-message-insert-unique'),
+    schema='augur_data',
+    comment='This table holds commit messages'
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('commit_messages', schema='augur_data')
+    # ### end Alembic commands ###

augur/tasks/data_analysis/contributor_breadth_worker/contributor_breadth_worker.py

@@ -4,7 +4,7 @@
 from datetime import datetime
 
 from augur.tasks.init.celery_app import celery_app as celery
-from augur.tasks.github.util.github_paginator import GithubPaginator
+from augur.tasks.github.util.github_data_access import GithubDataAccess, UrlNotFoundException
 from augur.application.db.models import ContributorRepo
 from augur.application.db.lib import bulk_insert_dicts
 from augur.tasks.github.util.github_random_key_auth import GithubRandomKeyAuth
@@ -83,6 +83,8 @@ def contributor_breadth_model(self) -> None:
 
         cntrb_newest_events_map[gh_login] = newest_event_date
 
+    github_data_access = GithubDataAccess(key_auth, logger)
+
     index = 1
     total = len(current_cntrb_logins)
     for cntrb in current_cntrb_logins:
@@ -98,18 +100,22 @@ def contributor_breadth_model(self) -> None:
 
 
         cntrb_events = []
-        for page_data, page in GithubPaginator(repo_cntrb_url, key_auth, logger).iter_pages():
+        try:
+            for event in github_data_access.paginate_resource(repo_cntrb_url):
 
-            if page_data: 
-                cntrb_events += page_data
+                cntrb_events.append(event)
 
-                oldest_event_on_page = datetime.strptime(page_data[-1]["created_at"], "%Y-%m-%dT%H:%M:%SZ")
-                if oldest_event_on_page < newest_event_in_db:
-                    print("Found cntrb events we already have...skipping the rest")
+                event_age = datetime.strptime(event["created_at"], "%Y-%m-%dT%H:%M:%SZ")
+                if event_age < newest_event_in_db:
+                    logger.info("Found cntrb events we already have...skipping the rest")
                     break
 
-        if len(cntrb_events) == 0:
-            logger.info("There are no cntrb events, or new events for this user.\n")
+            if len(cntrb_events) == 0:
+                logger.info("There are no cntrb events, or new events for this user.\n")
+                continue
+
+        except UrlNotFoundException as e:
+            logger.warning(e)
             continue
 
         events = process_contributor_events(cntrb, cntrb_events, logger, tool_source, tool_version, data_source)