Skip to content
This repository was archived by the owner on Sep 25, 2019. It is now read-only.

Commit 02a838d

Browse files
committed
Merge pull request #57 from gdg-x/fixClusterResourceLeak
fix(cluster): fix server outages due to cluster resource leakage
2 parents e6aa9e6 + 227f519 commit 02a838d

File tree

7 files changed

+73
-50
lines changed

7 files changed

+73
-50
lines changed

lib/.jshintrc

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,14 +2,21 @@
22
"node": true,
33
"esnext": true,
44
"bitwise": true,
5+
"camelcase": true,
6+
"curly": true,
57
"eqeqeq": true,
68
"immed": true,
7-
"latedef": true,
9+
"indent": 2,
10+
"latedef": "nofunc",
811
"newcap": true,
912
"noarg": true,
13+
"quotmark": "single",
1014
"regexp": true,
1115
"undef": true,
16+
"unused": true,
1217
"smarttabs": true,
18+
"strict": true,
19+
"trailing": true,
1320
"globals": {
1421
"next": false,
1522
"document": false

lib/config/keys.js

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -3,14 +3,13 @@
33
module.exports = {
44
keys: {
55
google: {
6-
simpleApiKey: process.env.GOOGLE_SIMPLE_API_KEY || "",
7-
oauthClientId: process.env.GOOGLE_OAUTH_CLIENT_ID || "",
8-
oauthClientSecret: process.env.GOOGLE_OAUTH_CLIENT_SECRET || ""
6+
simpleApiKey: process.env.GOOGLE_SIMPLE_API_KEY || '',
7+
oauthClientId: process.env.GOOGLE_OAUTH_CLIENT_ID || '',
8+
oauthClientSecret: process.env.GOOGLE_OAUTH_CLIENT_SECRET || ''
99
},
10-
newrelic: {
11-
licenseKey: ""
10+
frisbee: {
11+
serverClientId: process.env.SERVER_KEY_SECRET || '',
12+
androidClientIds: process.env.ANDROID_CLIENT_IDS || []
1213
}
1314
}
1415
};
15-
16-
process.env.NEWRELIC_AGENT = module.exports.keys.newrelic.licenseKey;

lib/config/passport.js

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,13 +3,11 @@
33
var mongoose = require('mongoose'),
44
User = mongoose.model('User'),
55
passport = require('passport'),
6-
SimpleApiKey = mongoose.model('SimpleApiKey'),
76
OauthConsumer = mongoose.model('OauthConsumer'),
87
OauthClient = mongoose.model('OauthClient'),
98
ConsumerStrategy = require('passport-http-oauth').ConsumerStrategy,
109
TokenStrategy = require('passport-http-oauth').TokenStrategy,
1110
BearerStrategy = require('passport-http-bearer').Strategy,
12-
request = require('superagent'),
1311
config = require('./config'),
1412
utils = require('../utils');
1513

@@ -98,8 +96,10 @@ module.exports = function () {
9896
function (token, done) {
9997
utils.getGoogleCert(function (certs) {
10098
utils.decodeAndVerifyJwt(token, certs, function (err, claims) {
99+
101100
if (claims.aud === config.keys.frisbee.serverClientId &&
102-
config.keys.frisbee.androidClientIds.indexOf(claims.azp) !== -1) {
101+
config.keys.frisbee.androidClientIds.indexOf(claims.azp) !== -1) {
102+
103103
User.findOne({_id: claims.sub}, function (err, user) {
104104
if (err) {
105105
return done(err);

lib/fixtures/countries.js

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,13 @@
11
'use strict';
22

3-
var config = require('../config/config'),
4-
mongoose = require('mongoose'),
5-
fs = require("fs"),
3+
var mongoose = require('mongoose'),
4+
fs = require('fs'),
65
Country = mongoose.model('Country');
76

87
module.exports = function () {
98
Country.count({}, function (err, count) {
109
if (count === 0) {
11-
fs.readFile(__dirname + "/countries.json", 'utf8', function (err, data) {
10+
fs.readFile(__dirname + '/countries.json', 'utf8', function (err, data) {
1211
if (err) {
1312
console.log('Error: ' + err);
1413
return;

lib/fixtures/index.js

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
'use strict';
22

33
module.exports = function () {
4-
require("fs").readdirSync(__dirname + '/').forEach(function (file) {
4+
require('fs').readdirSync(__dirname + '/').forEach(function (file) {
55
if (file.match(/.+\.js/g) !== null && file.match(/.+\.json/g) === null && file !== 'index.js') {
6-
require("./" + file)();
6+
require('./' + file)();
77
}
88
});
99
};

lib/risky.js

Lines changed: 26 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ module.exports = (function () {
2727
}
2828
}
2929

30-
console.log("was master: " + master + ", now master: " + iAmMaster);
30+
console.log('was master: ' + master + ', now master: ' + iAmMaster);
3131
master = iAmMaster;
3232
};
3333

@@ -37,11 +37,11 @@ module.exports = (function () {
3737

3838
var logentry = new TaskLog();
3939
logentry._id = id;
40-
logentry.task_type = type;
41-
logentry.started_at = moment(started);
42-
logentry.ended_at = ended;
43-
logentry.requested_by = myId;
44-
logentry.executed_by = executor;
40+
logentry.task_type = type; // jshint ignore:line
41+
logentry.started_at = moment(started); // jshint ignore:line
42+
logentry.ended_at = ended; // jshint ignore:line
43+
logentry.requested_by = myId; // jshint ignore:line
44+
logentry.executed_by = executor; // jshint ignore:line
4545

4646
if (err) {
4747
logentry.result = 1;
@@ -51,8 +51,9 @@ module.exports = (function () {
5151
}
5252
logentry.save();
5353

54-
if (tasks[id])
54+
if (tasks[id]) {
5555
delete tasks[id];
56+
}
5657

5758
cb(err, result, elapsed);
5859
};
@@ -63,16 +64,16 @@ module.exports = (function () {
6364
readyCount++;
6465
if (readyCount === 2) {
6566
startup = moment().valueOf();
66-
console.log("Risky is up. I'm " + myId);
67+
console.log('Risky is up. I\'m ' + myId);
6768
self.on('group:hello', function (data) {
68-
if (data.sender !== myId && data.type === "hi") {
69+
if (data.sender !== myId && data.type === 'hi') {
6970
if (responder) {
70-
console.log("Cancel masterResponder");
71+
console.log('Cancel masterResponder');
7172
clearTimeout(responder);
7273
responder = null;
7374
}
7475

75-
console.log(data.sender + " just said hi. Replying.");
76+
console.log(data.sender + ' just said hi. Replying.');
7677
cluster[data.sender] = {
7778
started: data.started,
7879
nextHeartbeat: moment().valueOf() + 10000
@@ -84,9 +85,9 @@ module.exports = (function () {
8485
type: 'hi_reply',
8586
started: startup
8687
});
87-
} else if (data.sender !== myId && data.type === "hi_reply") {
88+
} else if (data.sender !== myId && data.type === 'hi_reply') {
8889
if (responder) {
89-
console.log("Cancel masterResponder");
90+
console.log('Cancel masterResponder');
9091
clearTimeout(responder);
9192
responder = null;
9293
}
@@ -109,7 +110,7 @@ module.exports = (function () {
109110
var next = (data.nextHeartbeat - moment().valueOf()) + 2000;
110111

111112
cluster[data.sender].timeout = setTimeout(function () {
112-
console.log(data.sender + " has gone down...");
113+
console.log(data.sender + ' has gone down...');
113114
delete cluster[data.sender];
114115
evaluateMaster();
115116
}, next);
@@ -120,24 +121,24 @@ module.exports = (function () {
120121
if (!master) {
121122
if (taskHandler[data.type]) {
122123
// Offer to execute task
123-
console.log("Offering to execute task with type: " + data.type);
124+
console.log('Offering to execute task with type: ' + data.type);
124125
self.emit('group:taskoffer', {
125126
sender: myId,
126127
type: data.type,
127128
id: data.taskId
128129
});
129130
} else {
130-
console.log("Don't know how to execute task");
131+
console.log('Don\'t know how to execute task');
131132
}
132133
} else {
133-
console.log("I'm the master, not doing any tasks");
134+
console.log('I\'m the master, not doing any tasks');
134135
}
135136
});
136137

137138
self.on('group:taskoffer', function (data) {
138139
// First one to send an offer wins
139-
if (tasks[data.id] && tasks[data.id].state === "open") {
140-
tasks[data.id].state = "executing";
140+
if (tasks[data.id] && tasks[data.id].state === 'open') {
141+
tasks[data.id].state = 'executing';
141142
tasks[data.id].executor = data.sender;
142143

143144
tasks[data.id].acceptCallback(null, data.id, data.type, moment());
@@ -154,7 +155,7 @@ module.exports = (function () {
154155

155156
self.on('group:taskstart', function (data) {
156157
if (data.recipient === myId) {
157-
console.log("Executing task... type: " + data.type + ", id: " + data.id);
158+
console.log('Executing task... type: ' + data.type + ', id: ' + data.id);
158159
taskHandler[data.type](data.id, data.params, function (err, result) {
159160
self.emit('group:taskdone', {
160161
sender: myId,
@@ -238,11 +239,11 @@ module.exports = (function () {
238239

239240
var id = uuid.v4();
240241
if ((master || force) && Object.keys(cluster).length > 0) {
241-
console.log("Sending out task " + taskType + " with id " + id);
242+
console.log('Sending out task ' + taskType + ' with id ' + id);
242243
tasks[id] = {
243244
type: taskType,
244245
started: moment().valueOf(),
245-
state: "open",
246+
state: 'open',
246247
params: params,
247248
acceptCallback: acceptCallback,
248249
cb: cb
@@ -254,15 +255,15 @@ module.exports = (function () {
254255
taskId: id
255256
});
256257
} else if (Object.keys(cluster).length === 0 && taskHandler[taskType]) {
257-
console.log("Doing task myself...");
258+
console.log('Doing task myself...');
258259
acceptCallback(null, id, taskType, moment());
259260
var started = moment().valueOf();
260261
taskHandler[taskType](id, params, function (err, result) {
261262
taskDone(id, taskType, started, myId, err, result, cb);
262263
});
263264
} else {
264-
console.log("Not doing it...");
265-
acceptCallback("Not doing it");
265+
console.log('Not doing it...');
266+
acceptCallback('Not doing it');
266267
}
267268
},
268269
on: function (channel, handler, cb) {

server.js

Lines changed: 25 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -21,19 +21,39 @@ console.log = function () {
2121
}
2222
};
2323

24-
if (cluster.isMaster) {
24+
function initWorker(worker) {
25+
var listeners;
26+
27+
listeners = worker.process.listeners('exit')[0];
28+
var exit = listeners[Object.keys(listeners)[0]];
29+
30+
listeners = worker.process.listeners('disconnect')[0];
31+
var disconnect = listeners[Object.keys(listeners)[0]];
32+
33+
worker.process.removeListener('exit', exit);
34+
worker.process.once('exit', function(exitCode, signalCode) {
35+
if (worker.state !== 'disconnected') {
36+
disconnect();
37+
}
38+
exit(exitCode, signalCode);
39+
});
40+
}
2541

42+
if (cluster.isMaster) {
43+
var i;
2644
// Fork workers.
27-
for (var i = 0; i < numCPUs; i++) {
45+
for (i = 0; i < numCPUs; i++) {
2846
setTimeout(function () {
2947
var worker = cluster.fork();
48+
initWorker(worker);
3049
console.log('worker started, PID ' + worker.process.pid);
3150
}, (i + 1) * 5000); // jshint ignore:line
3251
}
3352

3453
cluster.on('exit', function (deadWorker, code, signal) {
3554
// Restart the worker
3655
var worker = cluster.fork();
56+
initWorker(worker);
3757

3858
// Note the process IDs
3959
var newPID = worker.process.pid;
@@ -43,17 +63,13 @@ if (cluster.isMaster) {
4363
console.log('worker ' + oldPID + ' died. Code: ' + code + ', Signal: ' + signal);
4464
console.log('worker ' + newPID + ' born.');
4565
});
46-
4766
} else {
4867
// Default node environment to development
4968
process.env.NODE_ENV = process.env.NODE_ENV || 'development';
5069

5170
// Application Config
5271
var config = require('./lib/config/config');
5372

54-
// Disable NewRelic for now.
55-
//var newrelic = require('newrelic');
56-
5773
// Connect to database
5874
var db = mongoose.connect(config.mongo.uri, config.mongo.options); // jshint ignore:line
5975

@@ -64,6 +80,7 @@ if (cluster.isMaster) {
6480
require(modelsPath + '/' + file);
6581
});
6682

83+
// Import static data
6784
require('./lib/fixtures')();
6885

6986
var risky = require('./lib/risky');
@@ -72,9 +89,9 @@ if (cluster.isMaster) {
7289
require('./lib/config/passport')();
7390

7491
if (config.env === 'production' && config.redis) {
75-
var myId = process.env.OPENSHIFT_GEAR_UUID + '';
92+
var myId = 'workerId';
7693
if (cluster.isWorker) {
77-
myId = process.env.OPENSHIFT_GEAR_UUID + '-' + cluster.worker.process.pid;
94+
myId = 'worker-' + cluster.worker.process.pid;
7895
}
7996
risky.connect({
8097
port: config.redis.port,

0 commit comments

Comments
 (0)