From 23d67ceb3be218348b2f6e81750829dff83e67e3 Mon Sep 17 00:00:00 2001 From: chrisLeeTW Date: Tue, 22 Jan 2019 23:21:40 +0800 Subject: [PATCH] feat: use thread pool for trigger job queue to prevent overloading. --- .../BitbucketBuildListener.java | 24 +-- .../BitbucketBuildTrigger.java | 154 ++++++++++++------ .../BitbucketPullRequestsBuilder.java | 21 ++- .../BitbucketRepository.java | 48 +++--- 4 files changed, 148 insertions(+), 99 deletions(-) diff --git a/src/main/java/bitbucketpullrequestbuilder/bitbucketpullrequestbuilder/BitbucketBuildListener.java b/src/main/java/bitbucketpullrequestbuilder/bitbucketpullrequestbuilder/BitbucketBuildListener.java index 808c835..be062dc 100644 --- a/src/main/java/bitbucketpullrequestbuilder/bitbucketpullrequestbuilder/BitbucketBuildListener.java +++ b/src/main/java/bitbucketpullrequestbuilder/bitbucketpullrequestbuilder/BitbucketBuildListener.java @@ -1,8 +1,6 @@ package bitbucketpullrequestbuilder.bitbucketpullrequestbuilder; -import bitbucketpullrequestbuilder.bitbucketpullrequestbuilder.bitbucket.cloud.CloudBitbucketCause; import hudson.Extension; -import hudson.model.AbstractBuild; import hudson.model.Job; import hudson.model.Run; import hudson.model.TaskListener; @@ -21,7 +19,7 @@ public class BitbucketBuildListener extends RunListener> { private static final Logger logger = Logger.getLogger(BitbucketBuildListener.class.getName()); @Override - public void onStarted(Run r, TaskListener listener) { + public void onStarted(Run r, TaskListener listener) { logger.fine("BitbucketBuildListener onStarted called."); BitbucketBuilds builds = builds(r); if (builds != null) { @@ -30,7 +28,7 @@ public void onStarted(Run r, TaskListener listener) { } @Override - public void onCompleted(Run r, @Nonnull TaskListener listener) { + public void onCompleted(Run r, @Nonnull TaskListener listener) { logger.fine("BitbucketBuildListener onCompleted called."); BitbucketBuilds builds = builds(r); if (builds != null) { @@ -40,20 +38,16 @@ public void onCompleted(Run r, @Nonnull TaskListener listener) { private BitbucketBuilds builds(Run r) { BitbucketBuildTrigger trigger = null; - if (r instanceof AbstractBuild) { - trigger = BitbucketBuildTrigger.getTrigger(((AbstractBuild) r).getProject()); - } else { - Job job = r.getParent(); - if (job instanceof ParameterizedJobMixIn.ParameterizedJob) { - - for (Trigger t : ((ParameterizedJobMixIn.ParameterizedJob) job).getTriggers().values()) { - if (t instanceof BitbucketBuildTrigger) { - trigger = (BitbucketBuildTrigger) t; - } + + Job job = r.getParent(); + if (job instanceof ParameterizedJobMixIn.ParameterizedJob) { + for (Trigger t : ((ParameterizedJobMixIn.ParameterizedJob) job).getTriggers().values()) { + if (t instanceof BitbucketBuildTrigger) { + trigger = (BitbucketBuildTrigger) t; } } } + return trigger == null ? null : trigger.getBuilder().getBuilds(); } - } diff --git a/src/main/java/bitbucketpullrequestbuilder/bitbucketpullrequestbuilder/BitbucketBuildTrigger.java b/src/main/java/bitbucketpullrequestbuilder/bitbucketpullrequestbuilder/BitbucketBuildTrigger.java index 1eff116..1c9a95f 100644 --- a/src/main/java/bitbucketpullrequestbuilder/bitbucketpullrequestbuilder/BitbucketBuildTrigger.java +++ b/src/main/java/bitbucketpullrequestbuilder/bitbucketpullrequestbuilder/BitbucketBuildTrigger.java @@ -1,45 +1,64 @@ package bitbucketpullrequestbuilder.bitbucketpullrequestbuilder; -import antlr.ANTLRException; +import static com.cloudbees.plugins.credentials.CredentialsMatchers.instanceOf; + +import java.io.IOException; +import java.net.URISyntaxException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.logging.Level; +import java.util.logging.Logger; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + import com.cloudbees.plugins.credentials.CredentialsProvider; import com.cloudbees.plugins.credentials.common.StandardListBoxModel; import com.cloudbees.plugins.credentials.common.StandardUsernamePasswordCredentials; import com.cloudbees.plugins.credentials.common.UsernamePasswordCredentials; +import com.cloudbees.plugins.credentials.domains.DomainRequirement; + +import org.apache.commons.lang.StringUtils; +import org.eclipse.jgit.transport.URIish; +import org.jenkinsci.Symbol; +import org.kohsuke.stapler.DataBoundConstructor; +import org.kohsuke.stapler.StaplerRequest; + +import antlr.ANTLRException; import hudson.Extension; -import hudson.model.*; +import hudson.model.Cause; +import hudson.model.CauseAction; +import hudson.model.Executor; +import hudson.model.Item; +import hudson.model.Job; +import hudson.model.ParameterDefinition; +import hudson.model.ParameterValue; +import hudson.model.ParametersAction; +import hudson.model.ParametersDefinitionProperty; import hudson.model.Queue; +import hudson.model.Result; +import hudson.model.Run; import hudson.model.queue.QueueTaskFuture; import hudson.plugins.git.RevisionParameterAction; +import hudson.security.ACL; import hudson.triggers.Trigger; import hudson.triggers.TriggerDescriptor; import hudson.util.ListBoxModel; import jenkins.model.Jenkins; import jenkins.model.ParameterizedJobMixIn; import net.sf.json.JSONObject; -import org.apache.commons.lang.StringUtils; -import org.eclipse.jgit.transport.URIish; -import org.jenkinsci.Symbol; -import org.kohsuke.stapler.DataBoundConstructor; -import org.kohsuke.stapler.StaplerRequest; - -import java.io.IOException; -import java.net.URISyntaxException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.logging.Level; -import java.util.logging.Logger; -import javax.annotation.Nonnull; -import javax.annotation.Nullable; - -import static com.cloudbees.plugins.credentials.CredentialsMatchers.instanceOf; /** * Created by nishio */ public class BitbucketBuildTrigger extends Trigger> { private static final Logger logger = Logger.getLogger(BitbucketBuildTrigger.class.getName()); + private static final ExecutorService pool = Executors.newFixedThreadPool(5); + private final String projectPath; private final String bitbucketServer; private final String cron; @@ -165,6 +184,7 @@ public boolean getApproveIfSuccess() { public boolean getCancelOutdatedJobs() { return cancelOutdatedJobs; } + /** * @return a phrase that when entered in a comment will trigger a new build */ @@ -173,21 +193,28 @@ public String getCommentTrigger() { } @Override - public void start(Job project, boolean newInstance) { + public void start(Job job, boolean newInstance) { + super.start(job, newInstance); + try { this.bitbucketPullRequestsBuilder = BitbucketPullRequestsBuilder.getBuilder(); - this.bitbucketPullRequestsBuilder.setProject(project); + this.bitbucketPullRequestsBuilder.setJob(job); this.bitbucketPullRequestsBuilder.setTrigger(this); this.bitbucketPullRequestsBuilder.setupBuilder(); - } catch(IllegalStateException e) { + } catch(Exception e) { logger.log(Level.SEVERE, "Can't start trigger", e); return; } - super.start(project, newInstance); } - public static BitbucketBuildTrigger getTrigger(AbstractProject project) { - Trigger trigger = project.getTrigger(BitbucketBuildTrigger.class); + public static BitbucketBuildTrigger getTrigger(Job job) { + if (!(job instanceof ParameterizedJobMixIn.ParameterizedJob)) { + return null; + } + + ParameterizedJobMixIn.ParameterizedJob pjob = (ParameterizedJobMixIn.ParameterizedJob) job; + + Trigger trigger = pjob.getTriggers().get(descriptor); return (BitbucketBuildTrigger)trigger; } @@ -195,16 +222,6 @@ public BitbucketPullRequestsBuilder getBuilder() { return this.bitbucketPullRequestsBuilder; } - private ParameterizedJobMixIn retrieveScheduleJob(final Job job) { - // TODO 1.621+ use standard method - return new ParameterizedJobMixIn() { - @Override - protected Job asJob() { - return job; - } - }; - } - public QueueTaskFuture startJob(BitbucketCause cause) { Map values = this.getDefaultParameters(); @@ -213,10 +230,19 @@ public QueueTaskFuture startJob(BitbucketCause cause) { abortRunningJobsThatMatch(cause); } - return retrieveScheduleJob(this.job).scheduleBuild2(0, - new CauseAction(cause), - new ParametersAction(new ArrayList(values.values())), - new RevisionParameterAction(cause.getSourceCommitHash(), getBitbucketRepoUrl(cause.getRepositoryOwner(), cause.getRepositoryName()))); + ParameterizedJobMixIn scheduledJob = new ParameterizedJobMixIn() { + @Override + protected Job asJob() { + return job; + } + }; + + return scheduledJob.scheduleBuild2( + this.getInstance().getQuietPeriod(), + new CauseAction(cause), + new ParametersAction(new ArrayList(values.values())), + new RevisionParameterAction(cause.getSourceCommitHash()) + ); } private URIish getBitbucketRepoUrl(String repoOwner, String repoName) { @@ -252,7 +278,7 @@ private void abortRunningJobsThatMatch(@Nonnull BitbucketCause bitbucketCause) { logger.fine("Looking for running jobs that match PR ID: " + bitbucketCause.getPullRequestId()); for (Object o : job.getBuilds()) { if (o instanceof Run) { - Run build = (Run) o; + Run build = (Run) o; if (build.isBuilding() && hasCauseFromTheSamePullRequest(build.getCauses(), bitbucketCause)) { logger.fine("Aborting build: " + build + " since PR is outdated"); setBuildDescription(build); @@ -266,7 +292,7 @@ private void abortRunningJobsThatMatch(@Nonnull BitbucketCause bitbucketCause) { } } - private void setBuildDescription(final Run build) { + private void setBuildDescription(final Run build) { try { build.setDescription("Aborting build by `Bitbucket Pullrequest Builder Plugin`: " + build + " since PR is outdated"); } catch (IOException e) { @@ -303,13 +329,17 @@ private Map getDefaultParameters() { @Override public void run() { - Job project = this.getBuilder().getProject(); - if (project instanceof AbstractProject && ((AbstractProject)project).isDisabled()) { - logger.fine("Build Skip."); - } else { - this.bitbucketPullRequestsBuilder.run(); + Job job = this.getBuilder().getJob(); + String name = job.getFullName(); + + if (!job.isBuildable()) { + logger.log(Level.FINE, "Build Skip for job - {0}.", name); + } else { + logger.log(Level.FINE, "running trigger for the job - {0}", name); + + pool.submit(new TriggerRunnable(this.getBuilder())); this.getDescriptor().save(); - } + } } @Override @@ -346,9 +376,31 @@ public boolean configure(StaplerRequest req, JSONObject json) throws FormExcepti public ListBoxModel doFillCredentialsIdItems() { return new StandardListBoxModel() - .withEmptySelection() - .withMatching(instanceOf(UsernamePasswordCredentials.class), - CredentialsProvider.lookupCredentials(StandardUsernamePasswordCredentials.class)); + .withEmptySelection() + .withMatching( + instanceOf(UsernamePasswordCredentials.class), + CredentialsProvider.lookupCredentials( + StandardUsernamePasswordCredentials.class, + (Item) null, + ACL.SYSTEM, + (DomainRequirement) null + ) + ); + } + } + + private static final class TriggerRunnable implements Runnable { + private final BitbucketPullRequestsBuilder builder; + + TriggerRunnable(BitbucketPullRequestsBuilder builder) { + this.builder = builder; + } + + @Override + public void run() { + synchronized (this) { + this.builder.run(); + } } } } diff --git a/src/main/java/bitbucketpullrequestbuilder/bitbucketpullrequestbuilder/BitbucketPullRequestsBuilder.java b/src/main/java/bitbucketpullrequestbuilder/bitbucketpullrequestbuilder/BitbucketPullRequestsBuilder.java index 9ef02f0..fa3acfa 100644 --- a/src/main/java/bitbucketpullrequestbuilder/bitbucketpullrequestbuilder/BitbucketPullRequestsBuilder.java +++ b/src/main/java/bitbucketpullrequestbuilder/bitbucketpullrequestbuilder/BitbucketPullRequestsBuilder.java @@ -19,7 +19,7 @@ */ public class BitbucketPullRequestsBuilder { private static final Logger logger = Logger.getLogger(BitbucketBuildTrigger.class.getName()); - private Job project; + private Job job; private BitbucketBuildTrigger trigger; private BitbucketRepository repository; private BitbucketBuilds builds; @@ -33,14 +33,13 @@ public void stop() { } public void run() { - logger.fine("Build Start."); this.repository.init(); Collection targetPullRequests = this.repository.getTargetPullRequests(); this.repository.addFutureBuildTasks(targetPullRequests); } public BitbucketPullRequestsBuilder setupBuilder() { - if (this.project == null || this.trigger == null) { + if (this.job == null || this.trigger == null) { throw new IllegalStateException(); } this.repository = new BitbucketRepository(this.trigger.getProjectPath(), this); @@ -49,18 +48,18 @@ public BitbucketPullRequestsBuilder setupBuilder() { return this; } - public void setProject(Job project) { - this.project = project; + public void setJob(Job job) { + this.job = job; } public void setTrigger(BitbucketBuildTrigger trigger) { this.trigger = trigger; } - public Job getProject() { - return this.project; - } - + public Job getJob() { + return this.job; + } + /** * Return MD5 hashed full project name or full project name, if MD5 hash provider inaccessible * @return unique project id @@ -68,13 +67,13 @@ public void setTrigger(BitbucketBuildTrigger trigger) { public String getProjectId() { try { final MessageDigest MD5 = MessageDigest.getInstance("MD5"); - return new String(Hex.encodeHex(MD5.digest(this.project.getFullName().getBytes("UTF-8")))); + return new String(Hex.encodeHex(MD5.digest(this.job.getFullName().getBytes("UTF-8")))); } catch (NoSuchAlgorithmException exc) { logger.log(Level.WARNING, "Failed to produce hash", exc); } catch (UnsupportedEncodingException exc) { logger.log(Level.WARNING, "Failed to produce hash", exc); } - return this.project.getFullName(); + return this.job.getFullName(); } diff --git a/src/main/java/bitbucketpullrequestbuilder/bitbucketpullrequestbuilder/BitbucketRepository.java b/src/main/java/bitbucketpullrequestbuilder/bitbucketpullrequestbuilder/BitbucketRepository.java index 60f574f..cee2244 100644 --- a/src/main/java/bitbucketpullrequestbuilder/bitbucketpullrequestbuilder/BitbucketRepository.java +++ b/src/main/java/bitbucketpullrequestbuilder/bitbucketpullrequestbuilder/BitbucketRepository.java @@ -1,38 +1,39 @@ package bitbucketpullrequestbuilder.bitbucketpullrequestbuilder; +import static com.cloudbees.plugins.credentials.CredentialsMatchers.instanceOf; + import java.util.ArrayList; import java.util.Collection; import java.util.Collections; -import java.util.List; -import java.util.logging.Logger; - -import bitbucketpullrequestbuilder.bitbucketpullrequestbuilder.bitbucket.ApiClient; -import bitbucketpullrequestbuilder.bitbucketpullrequestbuilder.bitbucket.AbstractPullrequest; -import bitbucketpullrequestbuilder.bitbucketpullrequestbuilder.bitbucket.BuildState; -import bitbucketpullrequestbuilder.bitbucketpullrequestbuilder.bitbucket.cloud.CloudBitbucketCause; -import bitbucketpullrequestbuilder.bitbucketpullrequestbuilder.bitbucket.cloud.CloudApiClient; - import java.util.LinkedList; +import java.util.List; import java.util.logging.Level; +import java.util.logging.Logger; import java.util.regex.Matcher; import java.util.regex.Pattern; -import bitbucketpullrequestbuilder.bitbucketpullrequestbuilder.bitbucket.server.ServerApiClient; -import bitbucketpullrequestbuilder.bitbucketpullrequestbuilder.bitbucket.server.ServerBitbucketCause; import com.cloudbees.plugins.credentials.CredentialsMatchers; import com.cloudbees.plugins.credentials.CredentialsProvider; import com.cloudbees.plugins.credentials.common.StandardUsernamePasswordCredentials; import com.cloudbees.plugins.credentials.common.UsernamePasswordCredentials; +import com.cloudbees.plugins.credentials.domains.DomainRequirement; +import org.apache.commons.lang.StringUtils; + +import bitbucketpullrequestbuilder.bitbucketpullrequestbuilder.bitbucket.AbstractPullrequest; +import bitbucketpullrequestbuilder.bitbucketpullrequestbuilder.bitbucket.ApiClient; +import bitbucketpullrequestbuilder.bitbucketpullrequestbuilder.bitbucket.BuildState; +import bitbucketpullrequestbuilder.bitbucketpullrequestbuilder.bitbucket.cloud.CloudApiClient; +import bitbucketpullrequestbuilder.bitbucketpullrequestbuilder.bitbucket.cloud.CloudBitbucketCause; +import bitbucketpullrequestbuilder.bitbucketpullrequestbuilder.bitbucket.server.ServerApiClient; +import bitbucketpullrequestbuilder.bitbucketpullrequestbuilder.bitbucket.server.ServerBitbucketCause; +import hudson.model.Item; +import hudson.security.ACL; import jenkins.model.Jenkins; import jenkins.scm.api.SCMSource; import jenkins.scm.api.SCMSourceOwner; import jenkins.scm.api.SCMSourceOwners; -import org.apache.commons.lang.StringUtils; - -import static com.cloudbees.plugins.credentials.CredentialsMatchers.instanceOf; - /** * Created by nishio */ @@ -47,13 +48,11 @@ public class BitbucketRepository { */ public static final String DEFAULT_COMMENT_TRIGGER = "test this please"; - private String projectPath; private BitbucketPullRequestsBuilder builder; private BitbucketBuildTrigger trigger; private ApiClient client; public BitbucketRepository(String projectPath, BitbucketPullRequestsBuilder builder) { - this.projectPath = projectPath; this.builder = builder; } @@ -208,7 +207,7 @@ public void setBuildStatus(BitbucketCause cause, BuildState state, String buildU logger.fine("setBuildStatus " + state + " for commit: " + sourceCommit + " with url " + buildUrl); if (state == BuildState.FAILED || state == BuildState.SUCCESSFUL) { - comment = String.format(BUILD_DESCRIPTION, builder.getProject().getDisplayName(), sourceCommit, destinationBranch); + comment = String.format(BUILD_DESCRIPTION, builder.getJob().getDisplayName(), sourceCommit, destinationBranch); } this.client.setBuildStatus(owner, repository, sourceCommit, state, buildUrl, comment, this.builder.getProjectId()); @@ -371,9 +370,14 @@ private boolean isFilteredBuild(AbstractPullrequest pullRequest) { private StandardUsernamePasswordCredentials getCredentials(String credentialsId) { if (null == credentialsId) return null; return CredentialsMatchers - .firstOrNull( - CredentialsProvider.lookupCredentials(StandardUsernamePasswordCredentials.class), - CredentialsMatchers.allOf(CredentialsMatchers.withId(credentialsId), - instanceOf(UsernamePasswordCredentials.class))); + .firstOrNull( + CredentialsProvider.lookupCredentials( + StandardUsernamePasswordCredentials.class, + (Item) null, + ACL.SYSTEM, + (DomainRequirement) null + ), + CredentialsMatchers.allOf(CredentialsMatchers.withId(credentialsId), instanceOf(UsernamePasswordCredentials.class)) + ); } }