diff --git a/treeherder/etl/management/commands/ingest.py b/treeherder/etl/management/commands/ingest.py index a4c6b6a066c..1a28cf38bde 100644 --- a/treeherder/etl/management/commands/ingest.py +++ b/treeherder/etl/management/commands/ingest.py @@ -259,7 +259,7 @@ def repo_meta(project): } -def query_data(repo_meta, commit): +def query_data(repo_meta, commit, branch): """Find the right event base sha to get the right list of commits This is not an issue in GithubPushTransformer because the PushEvent from Taskcluster @@ -271,7 +271,7 @@ def query_data(repo_meta, commit): # First we try with `master` being the base sha # e.g. https://api.github.com/repos/servo/servo/compare/master...1418c0555ff77e5a3d6cf0c6020ba92ece36be2e compare_response = github.compare_shas( - repo_meta["owner"], repo_meta["repo"], repo_meta["branch"], commit + repo_meta["owner"], repo_meta["repo"], branch or repo_meta["branch"], commit ) merge_base_commit = compare_response.get("merge_base_commit") if merge_base_commit: @@ -319,8 +319,8 @@ def query_data(repo_meta, commit): return event_base_sha, commits -def github_push_to_pulse(repo_meta, commit): - event_base_sha, commits = query_data(repo_meta, commit) +def github_push_to_pulse(repo_meta, commit, branch): + event_base_sha, commits = query_data(repo_meta, commit, branch) return { "exchange": "exchange/taskcluster-github/v1/push", @@ -329,7 +329,7 @@ def github_push_to_pulse(repo_meta, commit): "organization": repo_meta["owner"], "details": { "event.head.repo.url": "{}.git".format(repo_meta["url"]), - "event.base.repo.branch": repo_meta["branch"], + "event.base.repo.branch": branch or repo_meta["branch"], "event.base.sha": event_base_sha, "event.head.sha": commit, }, @@ -341,10 +341,10 @@ def github_push_to_pulse(repo_meta, commit): } -def ingest_push(project, revision, fetch_push_id=None): +def ingest_push(project, revision, branch=None, fetch_push_id=None): _repo = repo_meta(project) if _repo["url"].startswith("https://github.com"): - pulse = github_push_to_pulse(_repo, revision) + pulse = github_push_to_pulse(_repo, revision, branch) PushLoader().process(pulse["payload"], pulse["exchange"], _repo["tc_root_url"]) else: _ingest_hg_push(project, revision) @@ -417,6 +417,11 @@ def add_arguments(self, parser): "ingestion_type", nargs=1, help="Type of ingestion to do: [task|hg-push|git-commit|pr]" ) parser.add_argument("-p", "--project", help="Hg repository to query (e.g. autoland)") + parser.add_argument( + "-b", + "--branch", + help="branch to query (e.g. release/vXXX) - required if repository accepts more than one branch", + ) parser.add_argument("-c", "--commit", "-r", "--revision", help="Commit/revision to import") parser.add_argument( "--enable-eager-celery", @@ -474,7 +479,7 @@ def handle(self, *args, **options): "If you don't set up GITHUB_TOKEN you might hit Github's rate limiting. See docs for info." ) if type_of_ingestion == "git-push": - ingest_push(options["project"], options["commit"]) + ingest_push(options["project"], options["commit"], options["branch"]) elif type_of_ingestion == "git-pushes": ingest_git_pushes(options["project"], options["dryRun"]) elif type_of_ingestion == "push": diff --git a/treeherder/etl/push_loader.py b/treeherder/etl/push_loader.py index 05e3a67a5ae..e1e1ebaaf96 100644 --- a/treeherder/etl/push_loader.py +++ b/treeherder/etl/push_loader.py @@ -24,7 +24,13 @@ def process(self, message_body, exchange, root_url): newrelic.agent.add_custom_attribute("branch", transformer.branch) repos = Repository.objects if transformer.branch: - repos = repos.filter(branch__regex=f"(^|,){transformer.branch}($|,)") + if "/" in transformer.branch: + branch_domain = transformer.branch.split("/")[0] + "\\/\\*" + repos = repos.filter( + branch__regex=f"(^|,){transformer.branch}($|,)|(^|,){branch_domain}($|,)" + ) + else: + repos = repos.filter(branch__regex=f"(^|,){transformer.branch}($|,)") else: repos = repos.filter(branch=None) repo = repos.get(url=transformer.repo_url, active_status="active") diff --git a/treeherder/model/fixtures/repository.json b/treeherder/model/fixtures/repository.json index dbbec6ee53a..e796e6ed444 100644 --- a/treeherder/model/fixtures/repository.json +++ b/treeherder/model/fixtures/repository.json @@ -1852,7 +1852,7 @@ "dvcs_type": "git", "name": "firefox-ios", "url": "https://github.com/mozilla-mobile/firefox-ios", - "branch": "main", + "branch": "main,release/*", "active_status": "active", "codebase": "firefox-ios", "repository_group": 11, @@ -1954,7 +1954,7 @@ "dvcs_type": "git", "name": "staging-firefox-ios", "url": "https://github.com/mozilla-mobile/staging-firefox-ios", - "branch": "main", + "branch": "main,release/*", "active_status": "active", "codebase": "firefox-ios", "repository_group": 11,