Skip to content

Correct trailing symbol parsing #149

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
nfrasser merged 3 commits into from
Jul 17, 2016
Merged

Correct trailing symbol parsing #149

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
72b3dbc
Scanner enhancements
Jul 17, 2016
c8c8299
Parse trailing symbols in URLs more greedily
Jul 17, 2016
423304d
Significantly better default transition code
Jul 17, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 8 additions & 8 deletions src/linkify/core/parser.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ S_TLD = makeState(T_URL), // (A) Simplest possible URL with no query string
S_TLD_COLON = makeState(), // (A) URL followed by colon (potential port number here)
S_TLD_PORT = makeState(T_URL), // TLD followed by a port number
S_URL = makeState(T_URL), // Long URL with optional port and maybe query string
S_URL_SYMS = makeState(), // URL followed by some symbols (will not be part of the final URL)
S_URL_NON_ACCEPTING = makeState(), // URL followed by some symbols (will not be part of the final URL)
S_URL_OPENBRACE = makeState(), // URL followed by {
S_URL_OPENBRACKET = makeState(), // URL followed by [
S_URL_OPENPAREN = makeState(), // URL followed by (
Expand Down Expand Up @@ -152,7 +152,8 @@ let qsAccepting = [
TT_POUND,
TT_PROTOCOL,
TT_SLASH,
TT_TLD
TT_TLD,
TT_SYM
];

// Types of tokens that can follow a URL and be part of the query string
Expand All @@ -168,8 +169,7 @@ let qsNonAccepting = [
TT_CLOSEPAREN,
TT_OPENBRACE,
TT_OPENBRACKET,
TT_OPENPAREN,
TT_SYM
TT_OPENPAREN
];

// These states are responsible primarily for determining whether or not to
Expand All @@ -182,7 +182,7 @@ S_URL
.on(TT_OPENPAREN, S_URL_OPENPAREN);

// URL with extra symbols at the end, followed by an opening bracket
S_URL_SYMS
S_URL_NON_ACCEPTING
.on(TT_OPENBRACE, S_URL_OPENBRACE)
.on(TT_OPENBRACKET, S_URL_OPENBRACKET)
.on(TT_OPENPAREN, S_URL_OPENPAREN);
Expand Down Expand Up @@ -225,10 +225,10 @@ S_URL_OPENPAREN_SYMS.on(qsNonAccepting, S_URL_OPENPAREN_SYMS);

// Account for the query string
S_URL.on(qsAccepting, S_URL);
S_URL_SYMS.on(qsAccepting, S_URL);
S_URL_NON_ACCEPTING.on(qsAccepting, S_URL);

S_URL.on(qsNonAccepting, S_URL_SYMS);
S_URL_SYMS.on(qsNonAccepting, S_URL_SYMS);
S_URL.on(qsNonAccepting, S_URL_NON_ACCEPTING);
S_URL_NON_ACCEPTING.on(qsNonAccepting, S_URL_NON_ACCEPTING);

// Email address-specific state definitions
// Note: We are not allowing '/' in email addresses since this would interfere
Expand Down
36 changes: 18 additions & 18 deletions src/linkify/core/scanner.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,10 +12,10 @@ import {CharacterState as State, stateify} from './state';

const tlds = __TLDS__; // macro, see gulpfile.js

const
REGEXP_NUM = /[0-9]/,
REGEXP_ALPHANUM = /[a-z0-9]/,
COLON = ':';
const NUM = '0123456789'.split('');
const ALPHANUM = '0123456789abcdefghijklmnopqrstuvwxyz'.split('');
const WHITESPACE = [' ', '\f', '\r', '\t', '\v']; // excluding line breaks
const COLON = ':';

let
domainStates = [], // states that jump to DOMAIN on /[a-z0-9]/
Expand All @@ -30,7 +30,7 @@ T_TLD = TOKENS.TLD,
T_WS = TOKENS.WS;

const // Frequently used states
S_START = makeState(), // start state
S_START = makeState(),
S_NUM = makeState(T_NUM),
S_DOMAIN = makeState(T_DOMAIN),
S_DOMAIN_HYPHEN = makeState(), // domain followed by 1 or more hyphen characters
Expand All @@ -51,16 +51,16 @@ S_START
.on('}', makeState(TOKENS.CLOSEBRACE))
.on(']', makeState(TOKENS.CLOSEBRACKET))
.on(')', makeState(TOKENS.CLOSEPAREN))
.on(/[,;!]/, makeState(TOKENS.PUNCTUATION));
.on([',', ';', '!', '"'], makeState(TOKENS.PUNCTUATION));

// Whitespace jumps
// Tokens of only non-newline whitespace are arbitrarily long
S_START
.on(/\n/, makeState(TOKENS.NL))
.on(/\s/, S_WS);
.on('\n', makeState(TOKENS.NL))
.on(WHITESPACE, S_WS);

// If any whitespace except newline, more whitespace!
S_WS.on(/[^\S\n]/, S_WS);
S_WS.on(WHITESPACE, S_WS);

// Generates states for top-level domains
// Note that this is most accurate when tlds are in alphabetical order
Expand Down Expand Up @@ -109,30 +109,30 @@ domainStates.push.apply(domainStates, partialLocalhostStates);
// Everything else
// DOMAINs make more DOMAINs
// Number and character transitions
S_START.on(REGEXP_NUM, S_NUM);
S_START.on(NUM, S_NUM);
S_NUM
.on('-', S_DOMAIN_HYPHEN)
.on(REGEXP_NUM, S_NUM)
.on(REGEXP_ALPHANUM, S_DOMAIN); // number becomes DOMAIN
.on(NUM, S_NUM)
.on(ALPHANUM, S_DOMAIN); // number becomes DOMAIN

S_DOMAIN
.on('-', S_DOMAIN_HYPHEN)
.on(REGEXP_ALPHANUM, S_DOMAIN);
.on(ALPHANUM, S_DOMAIN);

// All the generated states should have a jump to DOMAIN
for (let i = 0; i < domainStates.length; i++) {
domainStates[i]
.on('-', S_DOMAIN_HYPHEN)
.on(REGEXP_ALPHANUM, S_DOMAIN);
.on(ALPHANUM, S_DOMAIN);
}

S_DOMAIN_HYPHEN
.on('-', S_DOMAIN_HYPHEN)
.on(REGEXP_NUM, S_DOMAIN)
.on(REGEXP_ALPHANUM, S_DOMAIN);
.on(NUM, S_DOMAIN)
.on(ALPHANUM, S_DOMAIN);

// Any other character is considered a single symbol token
S_START.on(/./, makeState(TOKENS.SYM));
// Set default transition
S_START.defaultTransition = makeState(TOKENS.SYM);

/**
Given a string, returns an array of TOKEN instances representing the
Expand Down
4 changes: 3 additions & 1 deletion src/linkify/core/state.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,8 @@ function createStateClass() {
*/
const BaseState = createStateClass();
BaseState.prototype = {
defaultTransition: false,

/**
@method constructor
@param {Class} tClass Pass in the kind of token to emit if there are
Expand Down Expand Up @@ -74,7 +76,7 @@ BaseState.prototype = {
}

// Nowhere left to jump!
return false;
return this.defaultTransition;
},

/**
Expand Down
10 changes: 5 additions & 5 deletions test/spec/linkify-html-test.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,9 +42,9 @@ describe('linkify-html', function () {
'The URL is <a href="http://google.com" class="linkified" target="_blank">google.com</a> and the email is <strong><a href="mailto:[email protected]" class="linkified">[email protected]</a></strong>',
'The URL is <span href="https://google.com" class="my-linkify-class" target="_parent" rel="nofollow" onclick="console.log(\'Hello World!\')">google.com</span> and the email is <strong><span href="mailto:[email protected]?subject=Hello%20from%20Linkify" class="my-linkify-class" target="_parent" rel="nofollow" onclick="console.log(\'Hello World!\')">[email protected]</span></strong>'
], [
'Super long maps URL https://www.google.ca/maps/@43.472082,-80.5426668,18z?hl=en, a #hash-tag, and an email: test."wut"[email protected]!',
'Super long maps URL <a href="https://www.google.ca/maps/@43.472082,-80.5426668,18z?hl=en" class="linkified" target="_blank">https://www.google.ca/maps/@43.472082,-80.5426668,18z?hl=en</a>, a #hash-tag, and an email: <a href="mailto:test.&quot;wut&quot;[email protected]" class="linkified">test."wut"[email protected]</a>!',
'Super long maps URL <span href="https://www.google.ca/maps/@43.472082,-80.5426668,18z?hl=en" class="my-linkify-class" target="_parent" rel="nofollow" onclick="console.log(\'Hello World!\')">https://www.google.ca/maps/@43.472082,-8…</span>, a #hash-tag, and an email: <span href="mailto:test.&quot;wut&quot;[email protected]?subject=Hello%20from%20Linkify" class="my-linkify-class" target="_parent" rel="nofollow" onclick="console.log(\'Hello World!\')">test."wut"[email protected]</span>!',
'Super long maps URL https://www.google.ca/maps/@43.472082,-80.5426668,18z?hl=en, a #hash-tag, and an email: [email protected]!',
'Super long maps URL <a href="https://www.google.ca/maps/@43.472082,-80.5426668,18z?hl=en" class="linkified" target="_blank">https://www.google.ca/maps/@43.472082,-80.5426668,18z?hl=en</a>, a #hash-tag, and an email: <a href="mailto:[email protected]" class="linkified">[email protected]</a>!',
'Super long maps URL <span href="https://www.google.ca/maps/@43.472082,-80.5426668,18z?hl=en" class="my-linkify-class" target="_parent" rel="nofollow" onclick="console.log(\'Hello World!\')">https://www.google.ca/maps/@43.472082,-8…</span>, a #hash-tag, and an email: <span href="mailto:[email protected]?subject=Hello%20from%20Linkify" class="my-linkify-class" target="_parent" rel="nofollow" onclick="console.log(\'Hello World!\')">[email protected]</span>!',
], [
'This link is already in an anchor tag <a href="#bro">google.com</a> LOL and this one <h1>isnt http://github.com</h1>',
'This link is already in an anchor tag <a href="#bro">google.com</a> LOL and this one <h1>isnt <a href="http://github.com" class="linkified" target="_blank">http://github.com</a></h1>',
Expand Down Expand Up @@ -87,8 +87,8 @@ describe('linkify-html', function () {
'2.The URL is google.com and the email is <strong>[email protected]</strong>',
'2.The URL is google.com and the email is <strong><a href="mailto:[email protected]" class="linkified">[email protected]</a></strong>'
], [
'3.Super long maps URL https://www.google.ca/maps/@43.472082,-80.5426668,18z?hl=en, a #hash-tag, and an email: test."wut"[email protected]!',
'3.Super long maps URL <a href="https://www.google.ca/maps/@43.472082,-80.5426668,18z?hl=en" class="linkified" target="_blank">https://www.google.ca/maps/@43.472082,-80.5426668,18z?hl=en</a>, a #hash-tag, and an email: <a href="mailto:test.&quot;wut&quot;[email protected]" class="linkified">test."wut"[email protected]</a>!'
'3.Super long maps URL https://www.google.ca/maps/@43.472082,-80.5426668,18z?hl=en, a #hash-tag, and an email: [email protected]!',
'3.Super long maps URL <a href="https://www.google.ca/maps/@43.472082,-80.5426668,18z?hl=en" class="linkified" target="_blank">https://www.google.ca/maps/@43.472082,-80.5426668,18z?hl=en</a>, a #hash-tag, and an email: <a href="mailto:[email protected]" class="linkified">[email protected]</a>!'
], [
'4a.This link is already in an anchor tag <a href="#bro">google.com</a> LOL and this one <h1>isnt http://github.com</h1>',
'4a.This link is already in an anchor tag <a href="#bro">google.com</a> LOL and this one <h1>isnt <a href="http://github.com" class="linkified" target="_blank">http://github.com</a></h1>'
Expand Down
10 changes: 5 additions & 5 deletions test/spec/linkify-string-test.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ describe('linkify-string', function () {
defaultProtocol: 'https',
linkAttributes: {
rel: 'nofollow',
onclick: 'javascript:;'
onclick: 'javascript:alert("Hello");'
},
format: function (val) {
return val.truncate(40);
Expand All @@ -57,12 +57,12 @@ describe('linkify-string', function () {
], [
'The URL is google.com and the email is [email protected]',
'The URL is <a href="http://google.com" class="linkified" target="_blank">google.com</a> and the email is <a href="mailto:[email protected]" class="linkified">[email protected]</a>',
'The URL is <span href="https://google.com" class="my-linkify-class" target="_parent" rel="nofollow" onclick="javascript:;">google.com</span> and the email is <span href="mailto:[email protected]?subject=Hello%20from%20Linkify" class="my-linkify-class" target="_parent" rel="nofollow" onclick="javascript:;">[email protected]</span>',
'The URL is <span href="https://google.com" class="my-linkify-class" target="_parent" rel="nofollow" onclick="javascript:alert(&quot;Hello&quot;);">google.com</span> and the email is <span href="mailto:[email protected]?subject=Hello%20from%20Linkify" class="my-linkify-class" target="_parent" rel="nofollow" onclick="javascript:alert(&quot;Hello&quot;);">[email protected]</span>',
'The URL is google.com and the email is [email protected]',
], [
'Super long maps URL https://www.google.ca/maps/@43.472082,-80.5426668,18z?hl=en, a #hash-tag, and an email: test."wut"[email protected]!\n',
'Super long maps URL <a href="https://www.google.ca/maps/@43.472082,-80.5426668,18z?hl=en" class="linkified" target="_blank">https://www.google.ca/maps/@43.472082,-80.5426668,18z?hl=en</a>, a #hash-tag, and an email: <a href="mailto:test.&quot;wut&quot;[email protected]" class="linkified">test."wut"[email protected]</a>!\n',
'Super long maps URL <span href="https://www.google.ca/maps/@43.472082,-80.5426668,18z?hl=en" class="my-linkify-class" target="_parent" rel="nofollow" onclick="javascript:;">https://www.google.ca/maps/@43.472082,-8…</span>, a #hash-tag, and an email: <span href="mailto:test.&quot;wut&quot;[email protected]?subject=Hello%20from%20Linkify" class="my-linkify-class" target="_parent" rel="nofollow" onclick="javascript:;">test."wut"[email protected]</span>!<br>\n',
'Super long maps URL https://www.google.ca/maps/@43.472082,-80.5426668,18z?hl=en, a #hash-tag, and an email: [email protected]!\n',
'Super long maps URL <a href="https://www.google.ca/maps/@43.472082,-80.5426668,18z?hl=en" class="linkified" target="_blank">https://www.google.ca/maps/@43.472082,-80.5426668,18z?hl=en</a>, a #hash-tag, and an email: <a href="mailto:[email protected]" class="linkified">[email protected]</a>!\n',
'Super long maps URL <span href="https://www.google.ca/maps/@43.472082,-80.5426668,18z?hl=en" class="my-linkify-class" target="_parent" rel="nofollow" onclick="javascript:alert(&quot;Hello&quot;);">https://www.google.ca/maps/@43.472082,-8…</span>, a #hash-tag, and an email: <span href="mailto:[email protected]?subject=Hello%20from%20Linkify" class="my-linkify-class" target="_parent" rel="nofollow" onclick="javascript:alert(&quot;Hello&quot;);">[email protected]</span>!<br>\n',
]
];

Expand Down
12 changes: 12 additions & 0 deletions test/spec/linkify/core/parser-test.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -138,6 +138,18 @@ var tests = [
'HTTP Auth URLs should work: http://username:[email protected]',
[TEXT, URL],
['HTTP Auth URLs should work: ', 'http://username:[email protected]']
], [
'Trailing equal symbol should work: http://example.com/foo/bar?token=CtFOYuk0wjiqvHZF==',
[TEXT, URL],
['Trailing equal symbol should work: ', 'http://example.com/foo/bar?token=CtFOYuk0wjiqvHZF==']
], [
'"https://surrounded.by.quotes/"',
[TEXT, URL, TEXT],
['"', 'https://surrounded.by.quotes/', '"']
], [
'More weird character in http://facebook.com/#aZ?/:@-._~!$&\'()*+,;= that URL',
[TEXT, URL, TEXT],
['More weird character in ', 'http://facebook.com/#aZ?/:@-._~!$&\'()*+,;=', ' that URL']
]
];

Expand Down