var Chunker = require('../utils/chunker');
-module.exports = function Tokenizer(data, minifyContext) {
- var chunker = new Chunker(data, '}', 128);
- var chunk = chunker.next();
- var flatBlock = /(^@(font\-face|page|\-ms\-viewport|\-o\-viewport|viewport)|\\@.+?)/;
+var flatBlock = /(^@(font\-face|page|\-ms\-viewport|\-o\-viewport|viewport)|\\@.+?)/;
- var whatsNext = function(context) {
- var cursor = context.cursor;
- var mode = context.mode;
- var closest;
+function Tokenizer(minifyContext) {
+ this.minifyContext = minifyContext;
+}
- if (chunk.length == context.cursor) {
- if (chunker.isEmpty())
- return null;
+Tokenizer.prototype.toTokens = function (data) {
+ var chunker = new Chunker(data, '}', 128);
+ if (chunker.isEmpty())
+ return [];
+
+ var context = {
+ cursor: 0,
+ mode: 'top',
+ chunker: chunker,
+ chunk: chunker.next(),
+ outer: this.minifyContext
+ };
- chunk = chunker.next();
- context.cursor = 0;
- }
+ return tokenize(context);
+};
- if (mode == 'body') {
- closest = chunk.indexOf('}', cursor);
- return closest > -1 ?
- [closest, 'bodyEnd'] :
- null;
+function extractProperties(string) {
+ return string
+ .replace(/\s{2,}/g, ' ')
+ .replace(/ ?: ?/g, ':')
+ .replace(/([\(,]) /g, '$1')
+ .replace(/ ([\),])/g, '$1')
+ .split(';')
+ .map(function (value) { return value.trim(); })
+ .filter(function (value) { return value.length > 0; });
+}
+
+function extractSelectors(string) {
+ return string
+ .replace(/(\s{2,}|\s)/g, ' ')
+ .replace(/, /g, ',')
+ .split(',')
+ .map(function (value) { return value.trim(); });
+}
+
+function extractBlock(string) {
+ return string
+ .replace(/\s/g, ' ')
+ .replace(/\s{2,}/g, ' ')
+ .trim();
+}
+
+function whatsNext(context) {
+ var mode = context.mode;
+ var chunk = context.chunk;
+ var closest;
+
+ if (chunk.length == context.cursor) {
+ if (context.chunker.isEmpty())
+ return null;
+
+ context.chunk = chunk = context.chunker.next();
+ context.cursor = 0;
+ }
+
+ if (mode == 'body') {
+ closest = chunk.indexOf('}', context.cursor);
+ return closest > -1 ?
+ [closest, 'bodyEnd'] :
+ null;
+ }
+
+ var nextSpecial = chunk.indexOf('@', context.cursor);
+ var nextEscape = chunk.indexOf('__ESCAPED_', context.cursor);
+ var nextBodyStart = chunk.indexOf('{', context.cursor);
+ var nextBodyEnd = chunk.indexOf('}', context.cursor);
+
+ closest = nextSpecial;
+ if (closest == -1 || (nextEscape > -1 && nextEscape < closest))
+ closest = nextEscape;
+ if (closest == -1 || (nextBodyStart > -1 && nextBodyStart < closest))
+ closest = nextBodyStart;
+ if (closest == -1 || (nextBodyEnd > -1 && nextBodyEnd < closest))
+ closest = nextBodyEnd;
+
+ if (closest == -1)
+ return;
+ if (nextEscape === closest)
+ return [closest, 'escape'];
+ if (nextBodyStart === closest)
+ return [closest, 'bodyStart'];
+ if (nextBodyEnd === closest)
+ return [closest, 'bodyEnd'];
+ if (nextSpecial === closest)
+ return [closest, 'special'];
+}
+
+function tokenize(context) {
+ var chunk = context.chunk;
+ var tokenized = [];
+
+ while (true) {
+ var next = whatsNext(context);
+ if (!next) {
+ var whatsLeft = context.chunk.substring(context.cursor);
+ if (whatsLeft.length > 0) {
+ tokenized.push(whatsLeft);
+ context.cursor += whatsLeft.length;
+ }
+ break;
}
- var nextSpecial = chunk.indexOf('@', context.cursor);
- var nextEscape = mode == 'top' ? chunk.indexOf('__ESCAPED_COMMENT_CLEAN_CSS', context.cursor) : -1;
- var nextBodyStart = chunk.indexOf('{', context.cursor);
- var nextBodyEnd = chunk.indexOf('}', context.cursor);
-
- closest = nextSpecial;
- if (closest == -1 || (nextEscape > -1 && nextEscape < closest))
- closest = nextEscape;
- if (closest == -1 || (nextBodyStart > -1 && nextBodyStart < closest))
- closest = nextBodyStart;
- if (closest == -1 || (nextBodyEnd > -1 && nextBodyEnd < closest))
- closest = nextBodyEnd;
-
- if (closest == -1)
- return;
- if (nextEscape === closest)
- return [closest, 'escape'];
- if (nextBodyStart === closest)
- return [closest, 'bodyStart'];
- if (nextBodyEnd === closest)
- return [closest, 'bodyEnd'];
- if (nextSpecial === closest)
- return [closest, 'special'];
- };
+ var nextSpecial = next[0];
+ var what = next[1];
+ var nextEnd;
+ var oldMode;
- var tokenize = function(context) {
- var tokenized = [];
+ chunk = context.chunk;
- context = context || { cursor: 0, mode: 'top' };
+ if (what == 'special') {
+ var firstOpenBraceAt = chunk.indexOf('{', nextSpecial);
+ var firstSemicolonAt = chunk.indexOf(';', nextSpecial);
+ var isSingle = firstSemicolonAt > -1 && (firstOpenBraceAt == -1 || firstSemicolonAt < firstOpenBraceAt);
+ if (isSingle) {
+ nextEnd = chunk.indexOf(';', nextSpecial + 1);
- while (true) {
- var next = whatsNext(context);
- if (!next) {
- var whatsLeft = chunk.substring(context.cursor);
- if (whatsLeft.length > 0) {
- tokenized.push(whatsLeft);
- context.cursor += whatsLeft.length;
- }
- break;
- }
+ var single = extractBlock(chunk.substring(context.cursor, nextEnd + 1));
+ tokenized.push(single);
- var nextSpecial = next[0];
- var what = next[1];
- var nextEnd;
- var oldMode;
-
- if (what == 'special') {
- var firstOpenBraceAt = chunk.indexOf('{', nextSpecial);
- var firstSemicolonAt = chunk.indexOf(';', nextSpecial);
- var isSingle = firstSemicolonAt > -1 && (firstOpenBraceAt == -1 || firstSemicolonAt < firstOpenBraceAt);
- if (isSingle) {
- nextEnd = chunk.indexOf(';', nextSpecial + 1);
- tokenized.push(chunk.substring(context.cursor, nextEnd + 1));
-
- context.cursor = nextEnd + 1;
- } else {
- nextEnd = chunk.indexOf('{', nextSpecial + 1);
- var block = chunk.substring(context.cursor, nextEnd).trim();
-
- var isFlat = flatBlock.test(block);
- oldMode = context.mode;
- context.cursor = nextEnd + 1;
- context.mode = isFlat ? 'body' : 'block';
- var specialBody = tokenize(context);
- context.mode = oldMode;
-
- tokenized.push({ block: block, body: specialBody });
- }
- } else if (what == 'escape') {
- nextEnd = chunk.indexOf('__', nextSpecial + 1);
- var escaped = chunk.substring(context.cursor, nextEnd + 2);
- tokenized.push(escaped);
-
- context.cursor = nextEnd + 2;
- } else if (what == 'bodyStart') {
- var selector = chunk.substring(context.cursor, nextSpecial).trim();
+ context.cursor = nextEnd + 1;
+ } else {
+ nextEnd = chunk.indexOf('{', nextSpecial + 1);
+ var block = chunk.substring(context.cursor, nextEnd).trim();
+ var isFlat = flatBlock.test(block);
oldMode = context.mode;
- context.cursor = nextSpecial + 1;
- context.mode = 'body';
- var body = tokenize(context);
- context.mode = oldMode;
-
- tokenized.push({ selector: selector, body: body });
- } else if (what == 'bodyEnd') {
- // extra closing brace at the top level can be safely ignored
- if (context.mode == 'top') {
- var at = context.cursor;
- var warning = chunk[context.cursor] == '}' ?
- 'Unexpected \'}\' in \'' + chunk.substring(at - 20, at + 20) + '\'. Ignoring.' :
- 'Unexpected content: \'' + chunk.substring(at, nextSpecial + 1) + '\'. Ignoring.';
+ context.cursor = nextEnd + 1;
+ context.mode = isFlat ? 'body' : 'block';
+ var specialBody = tokenize(context);
- minifyContext.warnings.push(warning);
- context.cursor = nextSpecial + 1;
- continue;
- }
+ if (typeof specialBody == 'string')
+ specialBody = extractProperties(specialBody);
- if (context.mode != 'block')
- tokenized = chunk.substring(context.cursor, nextSpecial);
+ context.mode = oldMode;
+ tokenized.push({ block: block, body: specialBody });
+ }
+ } else if (what == 'escape') {
+ nextEnd = chunk.indexOf('__', nextSpecial + 1);
+ var escaped = chunk.substring(context.cursor, nextEnd + 2);
+ tokenized.push(escaped);
+
+ context.cursor = nextEnd + 2;
+ } else if (what == 'bodyStart') {
+ var selector = extractSelectors(chunk.substring(context.cursor, nextSpecial));
+
+ oldMode = context.mode;
+ context.cursor = nextSpecial + 1;
+ context.mode = 'body';
+ var body = extractProperties(tokenize(context));
+
+ context.mode = oldMode;
+
+ tokenized.push({ selector: selector, body: body });
+ } else if (what == 'bodyEnd') {
+ // extra closing brace at the top level can be safely ignored
+ if (context.mode == 'top') {
+ var at = context.cursor;
+ var warning = chunk[context.cursor] == '}' ?
+ 'Unexpected \'}\' in \'' + chunk.substring(at - 20, at + 20) + '\'. Ignoring.' :
+ 'Unexpected content: \'' + chunk.substring(at, nextSpecial + 1) + '\'. Ignoring.';
+
+ context.outer.warnings.push(warning);
context.cursor = nextSpecial + 1;
-
- break;
+ continue;
}
- }
- return tokenized;
- };
+ if (context.mode != 'block')
+ tokenized = chunk.substring(context.cursor, nextSpecial);
- return {
- process: function() {
- return tokenize();
+ context.cursor = nextSpecial + 1;
+
+ break;
}
- };
-};
+ }
+
+ return tokenized;
+}
+
+module.exports = Tokenizer;
--- /dev/null
+var vows = require('vows');
+var assert = require('assert');
+var Tokenizer = require('../../lib/selectors/tokenizer');
+
+function tokenizerContext(config) {
+ var ctx = {};
+
+ function tokenized(target) {
+ return function (source) {
+ var tokenized = new Tokenizer({}).toTokens(source);
+ assert.deepEqual(target, tokenized);
+ };
+ }
+
+ for (var test in config) {
+ ctx[test] = {
+ topic: config[test][0],
+ tokenized: tokenized(config[test][1])
+ };
+ }
+
+ return ctx;
+}
+
+vows.describe(Tokenizer)
+ .addBatch(
+ tokenizerContext({
+ 'no content': [
+ '',
+ []
+ ],
+ 'an escaped content': [
+ '__ESCAPED_COMMENT_CLEAN_CSS0__',
+ ['__ESCAPED_COMMENT_CLEAN_CSS0__']
+ ],
+ 'an empty selector': [
+ 'a{}',
+ [{ selector: ['a'], body: [] }]
+ ],
+ 'an empty selector with whitespace': [
+ 'a{ \n }',
+ [{ selector: ['a'], body: [] }]
+ ],
+ 'a selector': [
+ 'a{color:red}',
+ [{ selector: ['a'], body: ['color:red'] }]
+ ],
+ 'a selector with whitespace': [
+ 'a {color:red;\n\ndisplay : block }',
+ [{ selector: ['a'], body: ['color:red', 'display:block'] }]
+ ],
+ 'a selector with whitespace in functions': [
+ 'a{color:rgba( 255, 255, 0, 0.5 )}',
+ [{ selector: ['a'], body: ['color:rgba(255,255,0,0.5)'] }]
+ ],
+ 'a selector with empty properties': [
+ 'a{color:red; ; ; ;}',
+ [{ selector: ['a'], body: ['color:red'] }]
+ ],
+ 'a double selector': [
+ 'a,\n\ndiv.class > p {color:red}',
+ [{ selector: ['a', 'div.class > p'], body: ['color:red'] }]
+ ],
+ 'two selectors': [
+ 'a{color:red}div{color:blue}',
+ [
+ { selector: ['a'], body: ['color:red'] },
+ { selector: ['div'], body: ['color:blue'] }
+ ]
+ ],
+ 'media query': [
+ '@media (min-width:980px){}',
+ [{ block: '@media (min-width:980px)', body: [] }]
+ ],
+ 'media query with selectors': [
+ '@media (min-width:980px){a{color:red}}',
+ [{ block: '@media (min-width:980px)', body: [{ selector: ['a'], body: ['color:red'] }] }]
+ ],
+ 'media query spanning more than one chunk': [
+ '@media only screen and (max-width:1319px) and (min--moz-device-pixel-ratio:1.5),only screen and (max-width:1319px) and (-moz-min-device-pixel-ratio:1.5){a{color:#000}}',
+ [{ block: '@media only screen and (max-width:1319px) and (min--moz-device-pixel-ratio:1.5),only screen and (max-width:1319px) and (-moz-min-device-pixel-ratio:1.5)', body: [{ selector: ['a'], body: ['color:#000'] }] }]
+ ],
+ 'font-face': [
+ '@font-face{font-family: fontName;font-size:12px}',
+ [{ block: '@font-face', body: ['font-family:fontName', 'font-size:12px'] }]
+ ],
+ 'charset': [
+ '@charset \'utf-8\';a{color:red}',
+ ['@charset \'utf-8\';', { selector: ['a'], body: ['color:red'] }]
+ ],
+ 'charset after a line break': [
+ '\n@charset \n\'utf-8\';',
+ ['@charset \'utf-8\';']
+ ]
+ })
+ )
+ .export(module);