[NodeConf.eu 2014] Scaling AB Testing on Netflix.com with Node.js
description
Transcript of [NodeConf.eu 2014] Scaling AB Testing on Netflix.com with Node.js
Scaling A/B testing on Netflix.com with
_________Alex Liu @stinkydofu
data driven product development
A
B
C
D
E
F
G
A
B
C
D
E
F
G
A
B
C
D
E
F
G
A
B
C
D
E
F
G
A
B
C
D
E
F
G
A
B
C
D
E
F
G
A
B
C
D
E
F
G
Test 1 Test 2 Test 3 Test 4 Test 5 Test 6 Test 7
2,097,152 unique experiences across seven tests
hundreds of new A/B tests per year
433518929550349486086117218185493567650…72061153709996
2105 566 685templates CSS JS
2.5M unique packages every week
problem: conditional dependencies
Packaging
oldSearch
app.js
newSearch
dep1 dep2 dep3 dep4 dep5
sub-dep sub-depsub-dep sub-dep sub-dep sub-dep
oldSearch
app.js
newSearch
dep1 dep2 dep3 dep4 dep5
sub-dep sub-depsub-dep sub-dep sub-dep sub-dep
app.js
import jquery from 'jquery'; import oldSearch from 'oldSearch'; import newSearch from 'newSearch';
export ...
oldSearch
app.js
newSearch
dep1 dep2 dep3 dep4 dep5
sub-dep sub-depsub-dep sub-dep sub-dep sub-dep
685 files…?
2.5M packages…?
oldSearch
app.js
newSearch
dep1 dep2 dep3 dep4 dep5
sub-dep sub-depsub-dep sub-dep sub-dep sub-dep
problem: conditional dependencies
requestbuild
require('nf-include-when')
/* * @includewhen rule.notInNewSearch */
oldSearch.js
/* * @includewhen rule.inNewSearch */
newSearch.js
var Rule = require('nf-rule-infrastructure'), inNewSearch;
inNewSearch = new Rule('inNewSearch', function(context, cb) { var test = context.abtests.get(1534); cb(test && test.cell(1)); });
module.exports = inNewSearch;
anatomy of a rule
require('nf-asset-registry')
import jquery from 'jquery'; import oldSearch from 'oldSearch'; import newSearch from 'newSearch';
export ...
app.js
newSearch.js
jquery
oldSearch.js
app.js
registry
"app.js": { "deps": [ "jquery", "oldSearch.js", "newSearch.js", ], "depsFull": [ "jquery", "oldSearchDep2.js", "oldSearchDep1.js", "oldSearch.js", "newSearchDep2.js", "newSearchDep1.js", "newSearch.js" ], "fileSize": "4.41 kB", "fileSizeFull": "120.52 kB" }
"newSearch.js": { "rule": "inNewSearch", "deps": [ "jquery", "newSearchDep2.js", "newSearchDep1.js", ], "depsFull": [ "jquery", "newSearchSubDep3.js", "newSearchSubDep2.js" "newSearchSubDep1.js" "newSearchDep2.js", "newSearchDep1.js" ], "fileSize": "10.41 kB", "fileSizeFull": "40.52 kB" }
nf-include-when
require('nf-packager')
var packager = require('nf-packager'), includeWhen = require('nf-include-when'), registries = require('nf-asset-registry');
function getScriptUrl() return packager.getPackageDefinition('app.js', registries, includeWhen); }
"app.js": { "deps": [ "jquery", "oldSearch.js", "newSearch.js", ], "depsFull": [ "jquery", "oldSearchDep2.js", "oldSearchDep1.js", "oldSearch.js", "newSearchDep2.js", "newSearchDep1.js", "newSearch.js" ], "fileSize": "4.41 kB", "fileSizeFull": "120.52 kB" }
Step 1: Get the full dependency tree for the requested package from the registry.
[ "jquery", /* no rule */ "oldSearchDep2.js", /* no rule */ "oldSearchDep1.js", /* no rule */ "oldSearch.js", /* rules.notInNewSearch */ "newSearchDep2.js", /* no rule */ "newSearchDep1.js”, /* no rule */ "newSearch.js" /* rules.inNewSearch */ ]
Step 2: Determine which files have rules.
[ "jquery", /* no rule */ "oldSearchDep2.js", /* no rule */ "oldSearchDep1.js", /* no rule */ "oldSearch.js", /* rules.notInNewSearch */ "newSearchDep2.js", /* no rule */ "newSearchDep1.js”, /* no rule */ "newSearch.js" /* rules.inNewSearch */ ]
Step 3: Run the rules. Filter out all deps that resolved false.
✓
[ "jquery", /* no rule */ "oldSearchDep2.js", /* no rule */ "oldSearchDep1.js", /* no rule */ "oldSearch.js", /* rules.notInNewSearch */ "newSearchDep2.js", /* no rule */ "newSearchDep1.js”, /* no rule */ "newSearch.js" /* rules.inNewSearch */ ]
Step 4: Filter out all extraneous sub deps.
✓
Step 5: Concatenate the files.
[ "jquery", /* no rule */ "newSearchDep2.js", /* no rule */ "newSearchDep1.js”, /* no rule */ "newSearch.js" /* rules.inNewSearch */ ]
buildjavascript
registry
request registry
rulespackager
Bonus Round
be creative with the registry
"account/bb/models/ratingHistoryModel.js": { "rule": null, "deps": [...], "depsFull": [...], "depsCount": { "underscore": 2, "backbone": 1, "jquery": 2, "common/requirejs-plugins.js": 4, "requirejs-text": 4, "utils/contextData.js": 1, "common/nfNamespace.js": 1 }, "hash": "dd23b163", "fileSize": "1.21 kB", "fileSizeFull": "173.04 kB" }
dependency counting
dependency pruning
file sizes
@import (reference) "/common/_nf_defs.less"; @import (reference) "/member/memberCore.less"; @import (reference) "/components/menu.less"; @import (reference) "/components/breadcrumbs.less";
@import modules
"account/containerResponsive.css": { "rule": null, "deps": [...], "depsFull": [...], "depsCount": [...], "hash": "65a431f3", "fileSize": "709 B", "fileSizeFull": "709 B", "css": { "selectors": 8, "declarationBlocks": 6, "declarations": 17, "mediaQueries": 3 } }
css analysis
the
best part
"cache": { "account/pin.js": "define('account/pin.js', ['member/memberC…", "account/bb/models/changePlanModel.js": "define('account/b…", "account/bb/models/ratingHistoryModel.js": "define('account…", "account/bb/models/viewingActivityModel.js": "define('account…", "account/bb/views/changePlanView.js": "define('account/bb/vi…", "account/bb/views/changePlanView.js": "define('account/bb/vi…", "account/bb/views/emailSubView.js": "define('account/bb/views…", "account/bb/views/viewingActivityView.js": "define('account…", "common/UITracking.js": "define('common/UITracking.js, ['me…", "common/UITrackingOverlay.js": "define('common/UITrackingOve…", … … …
css
mappings
javascript
templates templates
mappings
javascript
css
templates
mappings
javascript
css
UI Bundle
deploy UI bundles
anytime
never touch the file system
< 5ms package response times
Takeaways▶ leverage the server ▶ static analysis FTW ▶ divide and conquer with modules
Our Learnings
learn by doing
fail fastmove faster
“I have not failed.I’ve just found 10,000 waysthat won’t work.”
Thomas Edison
simplify
thanks! come find us!
Alex Liu @stinkydofu
Chris Saint-Amant @csaintamant
Micah Ransdell @mjr578
Kris Baxter @kristoferbaxter