Skip to content

feat: use thread pool for trigger job queue to prevent overloading. #184

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Jan 22, 2019
Merged

feat: use thread pool for trigger job queue to prevent overloading. #184

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 9 additions & 15 deletions .../java/bitbucketpullrequestbuilder/bitbucketpullrequestbuilder/BitbucketBuildListener.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,6 @@
package bitbucketpullrequestbuilder.bitbucketpullrequestbuilder;

import bitbucketpullrequestbuilder.bitbucketpullrequestbuilder.bitbucket.cloud.CloudBitbucketCause;
import hudson.Extension;
import hudson.model.AbstractBuild;
import hudson.model.Job;
import hudson.model.Run;
import hudson.model.TaskListener;
Expand All @@ -21,7 +19,7 @@ public class BitbucketBuildListener extends RunListener<Run<?, ?>> {
private static final Logger logger = Logger.getLogger(BitbucketBuildListener.class.getName());

@Override
public void onStarted(Run r, TaskListener listener) {
public void onStarted(Run<?,?> r, TaskListener listener) {
logger.fine("BitbucketBuildListener onStarted called.");
BitbucketBuilds builds = builds(r);
if (builds != null) {
Expand All @@ -30,7 +28,7 @@ public void onStarted(Run r, TaskListener listener) {
}

@Override
public void onCompleted(Run r, @Nonnull TaskListener listener) {
public void onCompleted(Run<?,?> r, @Nonnull TaskListener listener) {
logger.fine("BitbucketBuildListener onCompleted called.");
BitbucketBuilds builds = builds(r);
if (builds != null) {
Expand All @@ -40,20 +38,16 @@ public void onCompleted(Run r, @Nonnull TaskListener listener) {

private BitbucketBuilds builds(Run<?, ?> r) {
BitbucketBuildTrigger trigger = null;
if (r instanceof AbstractBuild) {
trigger = BitbucketBuildTrigger.getTrigger(((AbstractBuild) r).getProject());
} else {
Job job = r.getParent();
if (job instanceof ParameterizedJobMixIn.ParameterizedJob) {

for (Trigger<?> t : ((ParameterizedJobMixIn.ParameterizedJob) job).getTriggers().values()) {
if (t instanceof BitbucketBuildTrigger) {
trigger = (BitbucketBuildTrigger) t;
}

Job<?,?> job = r.getParent();
if (job instanceof ParameterizedJobMixIn.ParameterizedJob) {
for (Trigger<?> t : ((ParameterizedJobMixIn.ParameterizedJob) job).getTriggers().values()) {
if (t instanceof BitbucketBuildTrigger) {
trigger = (BitbucketBuildTrigger) t;
}
}
}

return trigger == null ? null : trigger.getBuilder().getBuilds();
}

}
154 changes: 103 additions & 51 deletions ...n/java/bitbucketpullrequestbuilder/bitbucketpullrequestbuilder/BitbucketBuildTrigger.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,45 +1,64 @@
package bitbucketpullrequestbuilder.bitbucketpullrequestbuilder;

import antlr.ANTLRException;
import static com.cloudbees.plugins.credentials.CredentialsMatchers.instanceOf;

import java.io.IOException;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.logging.Level;
import java.util.logging.Logger;

import javax.annotation.Nonnull;
import javax.annotation.Nullable;

import com.cloudbees.plugins.credentials.CredentialsProvider;
import com.cloudbees.plugins.credentials.common.StandardListBoxModel;
import com.cloudbees.plugins.credentials.common.StandardUsernamePasswordCredentials;
import com.cloudbees.plugins.credentials.common.UsernamePasswordCredentials;
import com.cloudbees.plugins.credentials.domains.DomainRequirement;

import org.apache.commons.lang.StringUtils;
import org.eclipse.jgit.transport.URIish;
import org.jenkinsci.Symbol;
import org.kohsuke.stapler.DataBoundConstructor;
import org.kohsuke.stapler.StaplerRequest;

import antlr.ANTLRException;
import hudson.Extension;
import hudson.model.*;
import hudson.model.Cause;
import hudson.model.CauseAction;
import hudson.model.Executor;
import hudson.model.Item;
import hudson.model.Job;
import hudson.model.ParameterDefinition;
import hudson.model.ParameterValue;
import hudson.model.ParametersAction;
import hudson.model.ParametersDefinitionProperty;
import hudson.model.Queue;
import hudson.model.Result;
import hudson.model.Run;
import hudson.model.queue.QueueTaskFuture;
import hudson.plugins.git.RevisionParameterAction;
import hudson.security.ACL;
import hudson.triggers.Trigger;
import hudson.triggers.TriggerDescriptor;
import hudson.util.ListBoxModel;
import jenkins.model.Jenkins;
import jenkins.model.ParameterizedJobMixIn;
import net.sf.json.JSONObject;
import org.apache.commons.lang.StringUtils;
import org.eclipse.jgit.transport.URIish;
import org.jenkinsci.Symbol;
import org.kohsuke.stapler.DataBoundConstructor;
import org.kohsuke.stapler.StaplerRequest;

import java.io.IOException;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.logging.Level;
import java.util.logging.Logger;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;

import static com.cloudbees.plugins.credentials.CredentialsMatchers.instanceOf;

/**
* Created by nishio
*/
public class BitbucketBuildTrigger extends Trigger<Job<?, ?>> {
private static final Logger logger = Logger.getLogger(BitbucketBuildTrigger.class.getName());
private static final ExecutorService pool = Executors.newFixedThreadPool(5);

private final String projectPath;
private final String bitbucketServer;
private final String cron;
Expand Down Expand Up @@ -165,6 +184,7 @@ public boolean getApproveIfSuccess() {
public boolean getCancelOutdatedJobs() {
return cancelOutdatedJobs;
}

/**
* @return a phrase that when entered in a comment will trigger a new build
*/
Expand All @@ -173,38 +193,35 @@ public String getCommentTrigger() {
}

@Override
public void start(Job<?, ?> project, boolean newInstance) {
public void start(Job<?, ?> job, boolean newInstance) {
super.start(job, newInstance);

try {
this.bitbucketPullRequestsBuilder = BitbucketPullRequestsBuilder.getBuilder();
this.bitbucketPullRequestsBuilder.setProject(project);
this.bitbucketPullRequestsBuilder.setJob(job);
this.bitbucketPullRequestsBuilder.setTrigger(this);
this.bitbucketPullRequestsBuilder.setupBuilder();
} catch(IllegalStateException e) {
} catch(Exception e) {
logger.log(Level.SEVERE, "Can't start trigger", e);
return;
}
super.start(project, newInstance);
}

public static BitbucketBuildTrigger getTrigger(AbstractProject project) {
Trigger trigger = project.getTrigger(BitbucketBuildTrigger.class);
public static BitbucketBuildTrigger getTrigger(Job<?, ?> job) {
if (!(job instanceof ParameterizedJobMixIn.ParameterizedJob)) {
return null;
}

ParameterizedJobMixIn.ParameterizedJob pjob = (ParameterizedJobMixIn.ParameterizedJob) job;

Trigger<?> trigger = pjob.getTriggers().get(descriptor);
return (BitbucketBuildTrigger)trigger;
}

public BitbucketPullRequestsBuilder getBuilder() {
return this.bitbucketPullRequestsBuilder;
}

private ParameterizedJobMixIn retrieveScheduleJob(final Job<?, ?> job) {
// TODO 1.621+ use standard method
return new ParameterizedJobMixIn() {
@Override
protected Job asJob() {
return job;
}
};
}

public QueueTaskFuture<?> startJob(BitbucketCause cause) {
Map<String, ParameterValue> values = this.getDefaultParameters();

Expand All @@ -213,10 +230,19 @@ public QueueTaskFuture<?> startJob(BitbucketCause cause) {
abortRunningJobsThatMatch(cause);
}

return retrieveScheduleJob(this.job).scheduleBuild2(0,
new CauseAction(cause),
new ParametersAction(new ArrayList(values.values())),
new RevisionParameterAction(cause.getSourceCommitHash(), getBitbucketRepoUrl(cause.getRepositoryOwner(), cause.getRepositoryName())));
ParameterizedJobMixIn scheduledJob = new ParameterizedJobMixIn() {
@Override
protected Job asJob() {
return job;
}
};

return scheduledJob.scheduleBuild2(
this.getInstance().getQuietPeriod(),
new CauseAction(cause),
new ParametersAction(new ArrayList<ParameterValue>(values.values())),
new RevisionParameterAction(cause.getSourceCommitHash())
);
}

private URIish getBitbucketRepoUrl(String repoOwner, String repoName) {
Expand Down Expand Up @@ -252,7 +278,7 @@ private void abortRunningJobsThatMatch(@Nonnull BitbucketCause bitbucketCause) {
logger.fine("Looking for running jobs that match PR ID: " + bitbucketCause.getPullRequestId());
for (Object o : job.getBuilds()) {
if (o instanceof Run) {
Run build = (Run) o;
Run<?,?> build = (Run<?,?>) o;
if (build.isBuilding() && hasCauseFromTheSamePullRequest(build.getCauses(), bitbucketCause)) {
logger.fine("Aborting build: " + build + " since PR is outdated");
setBuildDescription(build);
Expand All @@ -266,7 +292,7 @@ private void abortRunningJobsThatMatch(@Nonnull BitbucketCause bitbucketCause) {
}
}

private void setBuildDescription(final Run build) {
private void setBuildDescription(final Run<?,?> build) {
try {
build.setDescription("Aborting build by `Bitbucket Pullrequest Builder Plugin`: " + build + " since PR is outdated");
} catch (IOException e) {
Expand Down Expand Up @@ -303,13 +329,17 @@ private Map<String, ParameterValue> getDefaultParameters() {

@Override
public void run() {
Job<?,?> project = this.getBuilder().getProject();
if (project instanceof AbstractProject && ((AbstractProject)project).isDisabled()) {
logger.fine("Build Skip.");
} else {
this.bitbucketPullRequestsBuilder.run();
Job<?,?> job = this.getBuilder().getJob();
String name = job.getFullName();

if (!job.isBuildable()) {
logger.log(Level.FINE, "Build Skip for job - {0}.", name);
} else {
logger.log(Level.FINE, "running trigger for the job - {0}", name);

pool.submit(new TriggerRunnable(this.getBuilder()));
this.getDescriptor().save();
}
}
}

@Override
Expand Down Expand Up @@ -346,9 +376,31 @@ public boolean configure(StaplerRequest req, JSONObject json) throws FormExcepti

public ListBoxModel doFillCredentialsIdItems() {
return new StandardListBoxModel()
.withEmptySelection()
.withMatching(instanceOf(UsernamePasswordCredentials.class),
CredentialsProvider.lookupCredentials(StandardUsernamePasswordCredentials.class));
.withEmptySelection()
.withMatching(
instanceOf(UsernamePasswordCredentials.class),
CredentialsProvider.lookupCredentials(
StandardUsernamePasswordCredentials.class,
(Item) null,
ACL.SYSTEM,
(DomainRequirement) null
)
);
}
}

private static final class TriggerRunnable implements Runnable {
private final BitbucketPullRequestsBuilder builder;

TriggerRunnable(BitbucketPullRequestsBuilder builder) {
this.builder = builder;
}

@Override
public void run() {
synchronized (this) {
this.builder.run();
}
}
}
}
21 changes: 10 additions & 11 deletions ...bitbucketpullrequestbuilder/bitbucketpullrequestbuilder/BitbucketPullRequestsBuilder.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
*/
public class BitbucketPullRequestsBuilder {
private static final Logger logger = Logger.getLogger(BitbucketBuildTrigger.class.getName());
private Job<?, ?> project;
private Job<?, ?> job;
private BitbucketBuildTrigger trigger;
private BitbucketRepository repository;
private BitbucketBuilds builds;
Expand All @@ -33,14 +33,13 @@ public void stop() {
}

public void run() {
logger.fine("Build Start.");
this.repository.init();
Collection<AbstractPullrequest> targetPullRequests = this.repository.getTargetPullRequests();
this.repository.addFutureBuildTasks(targetPullRequests);
}

public BitbucketPullRequestsBuilder setupBuilder() {
if (this.project == null || this.trigger == null) {
if (this.job == null || this.trigger == null) {
throw new IllegalStateException();
}
this.repository = new BitbucketRepository(this.trigger.getProjectPath(), this);
Expand All @@ -49,32 +48,32 @@ public BitbucketPullRequestsBuilder setupBuilder() {
return this;
}

public void setProject(Job<?, ?> project) {
this.project = project;
public void setJob(Job<?, ?> job) {
this.job = job;
}

public void setTrigger(BitbucketBuildTrigger trigger) {
this.trigger = trigger;
}

public Job<?, ?> getProject() {
return this.project;
}
public Job<?, ?> getJob() {
return this.job;
}

/**
* Return MD5 hashed full project name or full project name, if MD5 hash provider inaccessible
* @return unique project id
*/
public String getProjectId() {
try {
final MessageDigest MD5 = MessageDigest.getInstance("MD5");
return new String(Hex.encodeHex(MD5.digest(this.project.getFullName().getBytes("UTF-8"))));
return new String(Hex.encodeHex(MD5.digest(this.job.getFullName().getBytes("UTF-8"))));
} catch (NoSuchAlgorithmException exc) {
logger.log(Level.WARNING, "Failed to produce hash", exc);
} catch (UnsupportedEncodingException exc) {
logger.log(Level.WARNING, "Failed to produce hash", exc);
}
return this.project.getFullName();
return this.job.getFullName();

}

Expand Down
Loading