From 530722b06016736248d8c984d1895b3c5333caa5 Mon Sep 17 00:00:00 2001 From: Jakub Pawlowicz Date: Sat, 5 Nov 2016 08:16:15 +0100 Subject: [PATCH] Rewrites tokenizer without escaping. This way it's much easier to handle parsing as it's a "one step" process instead of way more complex, legacy escaping. --- History.md | 1 + lib/text/comments-processor.js | 131 - lib/text/escape-store.js | 53 - lib/text/expressions-processor.js | 117 - lib/text/free-text-processor.js | 98 - lib/text/urls-processor.js | 75 - lib/tokenizer/extract-properties.js | 193 - lib/tokenizer/extract-selectors.js | 17 - lib/tokenizer/marker.js | 22 + lib/tokenizer/token.js | 13 + lib/tokenizer/tokenize.js | 629 ++-- lib/utils/input-source-map-tracker-2.js | 30 + test/text/comments-processor-test.js | 192 - test/text/escape-store-test.js | 69 - test/text/expressions-processor-test.js | 105 - test/text/free-text-processor-test.js | 100 - test/text/urls-processor-test.js | 145 - test/tokenizer/tokenize-test.js | 3540 ++++++++++++++++++ test/tokenizer/tokenizer-source-maps-test.js | 582 --- test/tokenizer/tokenizer-test.js | 407 -- 20 files changed, 3987 insertions(+), 2532 deletions(-) delete mode 100644 lib/text/comments-processor.js delete mode 100644 lib/text/escape-store.js delete mode 100644 lib/text/expressions-processor.js delete mode 100644 lib/text/free-text-processor.js delete mode 100644 lib/text/urls-processor.js delete mode 100644 lib/tokenizer/extract-properties.js delete mode 100644 lib/tokenizer/extract-selectors.js create mode 100644 lib/tokenizer/marker.js create mode 100644 lib/tokenizer/token.js create mode 100644 lib/utils/input-source-map-tracker-2.js delete mode 100644 test/text/comments-processor-test.js delete mode 100644 test/text/escape-store-test.js delete mode 100644 test/text/expressions-processor-test.js delete mode 100644 test/text/free-text-processor-test.js delete mode 100644 test/text/urls-processor-test.js create mode 100644 test/tokenizer/tokenize-test.js delete mode 100644 test/tokenizer/tokenizer-source-maps-test.js delete mode 100644 test/tokenizer/tokenizer-test.js diff --git a/History.md b/History.md index ed9dadee..9338edbf 100644 --- a/History.md +++ b/History.md @@ -2,6 +2,7 @@ ================== * Requires Node.js 4.0+ to run. +* Replaces the old tokenizer with a new one which doesn't use any escaping. [3.4.22 / 2016-12-12](https://github.com/jakubpawlowicz/clean-css/compare/v3.4.21...v3.4.22) ================== diff --git a/lib/text/comments-processor.js b/lib/text/comments-processor.js deleted file mode 100644 index 2063b53a..00000000 --- a/lib/text/comments-processor.js +++ /dev/null @@ -1,131 +0,0 @@ -var EscapeStore = require('./escape-store'); -var QuoteScanner = require('../utils/quote-scanner'); - -var SPECIAL_COMMENT_PREFIX = '/*!'; -var COMMENT_PREFIX = '/*'; -var COMMENT_SUFFIX = '*/'; - -var lineBreak = require('os').EOL; - -function CommentsProcessor(context, keepSpecialComments, keepBreaks, saveWaypoints) { - this.comments = new EscapeStore('COMMENT'); - this.specialComments = new EscapeStore('COMMENT_SPECIAL'); - - this.context = context; - this.restored = 0; - this.keepAll = keepSpecialComments == '*'; - this.keepOne = keepSpecialComments == '1' || keepSpecialComments === 1; - this.keepBreaks = keepBreaks; - this.saveWaypoints = saveWaypoints; -} - -function quoteScannerFor(data) { - var quoteMap = []; - new QuoteScanner(data).each(function (quotedString, _, startsAt) { - quoteMap.push([startsAt, startsAt + quotedString.length]); - }); - - return function (position) { - for (var i = 0, l = quoteMap.length; i < l; i++) { - if (quoteMap[i][0] < position && quoteMap[i][1] > position) - return true; - } - - return false; - }; -} - -CommentsProcessor.prototype.escape = function (data) { - var tempData = []; - var nextStart = 0; - var nextEnd = 0; - var cursor = 0; - var indent = 0; - var breaksCount; - var lastBreakAt; - var newIndent; - var isQuotedAt = quoteScannerFor(data); - var saveWaypoints = this.saveWaypoints; - - for (; nextEnd < data.length;) { - nextStart = data.indexOf(COMMENT_PREFIX, cursor); - if (nextStart == -1) - break; - - if (isQuotedAt(nextStart)) { - tempData.push(data.substring(cursor, nextStart + COMMENT_PREFIX.length)); - cursor = nextStart + COMMENT_PREFIX.length; - continue; - } - - nextEnd = data.indexOf(COMMENT_SUFFIX, nextStart + COMMENT_PREFIX.length); - if (nextEnd == -1) { - this.context.warnings.push('Broken comment: \'' + data.substring(nextStart) + '\'.'); - nextEnd = data.length - 2; - } - - tempData.push(data.substring(cursor, nextStart)); - - var comment = data.substring(nextStart, nextEnd + COMMENT_SUFFIX.length); - var isSpecialComment = comment.indexOf(SPECIAL_COMMENT_PREFIX) === 0; - - if (saveWaypoints) { - breaksCount = comment.split(lineBreak).length - 1; - lastBreakAt = comment.lastIndexOf(lineBreak); - newIndent = lastBreakAt > 0 ? - comment.substring(lastBreakAt + lineBreak.length).length : - indent + comment.length; - } - - if (saveWaypoints || isSpecialComment) { - var metadata = saveWaypoints ? [breaksCount, newIndent] : null; - var placeholder = isSpecialComment ? - this.specialComments.store(comment, metadata) : - this.comments.store(comment, metadata); - tempData.push(placeholder); - } - - if (saveWaypoints) - indent = newIndent + 1; - cursor = nextEnd + COMMENT_SUFFIX.length; - } - - return tempData.length > 0 ? - tempData.join('') + data.substring(cursor, data.length) : - data; -}; - -function restore(context, data, from, isSpecial) { - var tempData = []; - var cursor = 0; - - for (; cursor < data.length;) { - var nextMatch = from.nextMatch(data, cursor); - if (nextMatch.start < 0) - break; - - tempData.push(data.substring(cursor, nextMatch.start)); - var comment = from.restore(nextMatch.match); - - if (isSpecial && (context.keepAll || (context.keepOne && context.restored === 0))) { - context.restored++; - tempData.push(comment); - - cursor = nextMatch.end; - } else { - cursor = nextMatch.end + (context.keepBreaks && data.substring(nextMatch.end, nextMatch.end + lineBreak.length) == lineBreak ? lineBreak.length : 0); - } - } - - return tempData.length > 0 ? - tempData.join('') + data.substring(cursor, data.length) : - data; -} - -CommentsProcessor.prototype.restore = function (data) { - data = restore(this, data, this.comments, false); - data = restore(this, data, this.specialComments, true); - return data; -}; - -module.exports = CommentsProcessor; diff --git a/lib/text/escape-store.js b/lib/text/escape-store.js deleted file mode 100644 index 8598d732..00000000 --- a/lib/text/escape-store.js +++ /dev/null @@ -1,53 +0,0 @@ -var placeholderBrace = '__'; - -function EscapeStore(placeholderRoot) { - this.placeholderRoot = 'ESCAPED_' + placeholderRoot + '_CLEAN_CSS'; - this.placeholderToData = {}; - this.dataToPlaceholder = {}; - this.count = 0; - this.restoreMatcher = new RegExp(this.placeholderRoot + '(\\d+)'); -} - -EscapeStore.prototype._nextPlaceholder = function (metadata) { - return { - index: this.count, - value: placeholderBrace + this.placeholderRoot + this.count++ + metadata + placeholderBrace - }; -}; - -EscapeStore.prototype.store = function (data, metadata) { - var encodedMetadata = metadata ? - '(' + metadata.join(',') + ')' : - ''; - var placeholder = this.dataToPlaceholder[data]; - - if (!placeholder) { - var nextPlaceholder = this._nextPlaceholder(encodedMetadata); - placeholder = nextPlaceholder.value; - this.placeholderToData[nextPlaceholder.index] = data; - this.dataToPlaceholder[data] = nextPlaceholder.value; - } - - if (metadata) - placeholder = placeholder.replace(/\([^\)]+\)/, encodedMetadata); - - return placeholder; -}; - -EscapeStore.prototype.nextMatch = function (data, cursor) { - var next = {}; - - next.start = data.indexOf(this.placeholderRoot, cursor) - placeholderBrace.length; - next.end = data.indexOf(placeholderBrace, next.start + placeholderBrace.length) + placeholderBrace.length; - if (next.start > -1 && next.end > -1) - next.match = data.substring(next.start, next.end); - - return next; -}; - -EscapeStore.prototype.restore = function (placeholder) { - var index = this.restoreMatcher.exec(placeholder)[1]; - return this.placeholderToData[index]; -}; - -module.exports = EscapeStore; diff --git a/lib/text/expressions-processor.js b/lib/text/expressions-processor.js deleted file mode 100644 index 3e22ced8..00000000 --- a/lib/text/expressions-processor.js +++ /dev/null @@ -1,117 +0,0 @@ -var EscapeStore = require('./escape-store'); - -var EXPRESSION_NAME = 'expression'; -var EXPRESSION_START = '('; -var EXPRESSION_END = ')'; -var EXPRESSION_PREFIX = EXPRESSION_NAME + EXPRESSION_START; -var BODY_START = '{'; -var BODY_END = '}'; - -var lineBreak = require('os').EOL; - -function findEnd(data, start) { - var end = start + EXPRESSION_NAME.length; - var level = 0; - var quoted = false; - var braced = false; - - while (true) { - var current = data[end++]; - - if (quoted) { - quoted = current != '\'' && current != '"'; - } else { - quoted = current == '\'' || current == '"'; - - if (current == EXPRESSION_START) - level++; - if (current == EXPRESSION_END) - level--; - if (current == BODY_START) - braced = true; - if (current == BODY_END && !braced && level == 1) { - end--; - level--; - } - } - - if (level === 0 && current == EXPRESSION_END) - break; - if (!current) { - end = data.substring(0, end).lastIndexOf(BODY_END); - break; - } - } - - return end; -} - -function ExpressionsProcessor(saveWaypoints) { - this.expressions = new EscapeStore('EXPRESSION'); - this.saveWaypoints = saveWaypoints; -} - -ExpressionsProcessor.prototype.escape = function (data) { - var nextStart = 0; - var nextEnd = 0; - var cursor = 0; - var tempData = []; - var indent = 0; - var breaksCount; - var lastBreakAt; - var newIndent; - var saveWaypoints = this.saveWaypoints; - - for (; nextEnd < data.length;) { - nextStart = data.indexOf(EXPRESSION_PREFIX, nextEnd); - if (nextStart == -1) - break; - - nextEnd = findEnd(data, nextStart); - - var expression = data.substring(nextStart, nextEnd); - if (saveWaypoints) { - breaksCount = expression.split(lineBreak).length - 1; - lastBreakAt = expression.lastIndexOf(lineBreak); - newIndent = lastBreakAt > 0 ? - expression.substring(lastBreakAt + lineBreak.length).length : - indent + expression.length; - } - - var metadata = saveWaypoints ? [breaksCount, newIndent] : null; - var placeholder = this.expressions.store(expression, metadata); - tempData.push(data.substring(cursor, nextStart)); - tempData.push(placeholder); - - if (saveWaypoints) - indent = newIndent + 1; - cursor = nextEnd; - } - - return tempData.length > 0 ? - tempData.join('') + data.substring(cursor, data.length) : - data; -}; - -ExpressionsProcessor.prototype.restore = function (data) { - var tempData = []; - var cursor = 0; - - for (; cursor < data.length;) { - var nextMatch = this.expressions.nextMatch(data, cursor); - if (nextMatch.start < 0) - break; - - tempData.push(data.substring(cursor, nextMatch.start)); - var comment = this.expressions.restore(nextMatch.match); - tempData.push(comment); - - cursor = nextMatch.end; - } - - return tempData.length > 0 ? - tempData.join('') + data.substring(cursor, data.length) : - data; -}; - -module.exports = ExpressionsProcessor; diff --git a/lib/text/free-text-processor.js b/lib/text/free-text-processor.js deleted file mode 100644 index 8a426244..00000000 --- a/lib/text/free-text-processor.js +++ /dev/null @@ -1,98 +0,0 @@ -var EscapeStore = require('./escape-store'); -var QuoteScanner = require('../utils/quote-scanner'); - -var lineBreak = require('os').EOL; - -function FreeTextProcessor(saveWaypoints) { - this.matches = new EscapeStore('FREE_TEXT'); - this.saveWaypoints = saveWaypoints; -} - -// Strip content tags by replacing them by the a special -// marker for further restoring. It's done via string scanning -// instead of regexps to speed up the process. -FreeTextProcessor.prototype.escape = function (data) { - var self = this; - var breaksCount; - var lastBreakAt; - var indent; - var metadata; - var saveWaypoints = this.saveWaypoints; - - return new QuoteScanner(data).each(function (match, store) { - if (saveWaypoints) { - breaksCount = match.split(lineBreak).length - 1; - lastBreakAt = match.lastIndexOf(lineBreak); - indent = lastBreakAt > 0 ? - match.substring(lastBreakAt + lineBreak.length).length : - match.length; - metadata = [breaksCount, indent]; - } - - var placeholder = self.matches.store(match, metadata); - store.push(placeholder); - }); -}; - -function normalize(text, data, prefixContext, cursor) { - // FIXME: this is even a bigger hack now - see #407 - var searchIn = data; - if (prefixContext) { - searchIn = prefixContext + data.substring(0, data.indexOf('__ESCAPED_FREE_TEXT_CLEAN_CSS')); - cursor = searchIn.length; - } - - var lastSemicolon = searchIn.lastIndexOf(';', cursor); - var lastOpenBrace = searchIn.lastIndexOf('{', cursor); - var lastOne = 0; - - if (lastSemicolon > -1 && lastOpenBrace > -1) - lastOne = Math.max(lastSemicolon, lastOpenBrace); - else if (lastSemicolon == -1) - lastOne = lastOpenBrace; - else - lastOne = lastSemicolon; - - var context = searchIn.substring(lastOne + 1, cursor); - - if (/\[[\w\d\-]+[\*\|\~\^\$]?=$/.test(context)) { - text = text - .replace(/\\\n|\\\r\n/g, '') - .replace(/\n|\r\n/g, ''); - } - - if (/^['"][a-zA-Z][a-zA-Z\d\-_]+['"]$/.test(text) && !/format\($/.test(context)) { - var isFont = /^(font|font\-family):/.test(context); - var isAttribute = /\[[\w\d\-]+[\*\|\~\^\$]?=$/.test(context); - var isKeyframe = /@(-moz-|-o-|-webkit-)?keyframes /.test(context); - var isAnimation = /^(-moz-|-o-|-webkit-)?animation(-name)?:/.test(context); - - if (isFont || isAttribute || isKeyframe || isAnimation) - text = text.substring(1, text.length - 1); - } - - return text; -} - -FreeTextProcessor.prototype.restore = function (data, prefixContext) { - var tempData = []; - var cursor = 0; - - for (; cursor < data.length;) { - var nextMatch = this.matches.nextMatch(data, cursor); - if (nextMatch.start < 0) - break; - - tempData.push(data.substring(cursor, nextMatch.start)); - var text = normalize(this.matches.restore(nextMatch.match), data, prefixContext, nextMatch.start); - tempData.push(text); - - cursor = nextMatch.end; - } - - return tempData.length > 0 ? - tempData.join('') + data.substring(cursor, data.length) : - data; -}; - -module.exports = FreeTextProcessor; diff --git a/lib/text/urls-processor.js b/lib/text/urls-processor.js deleted file mode 100644 index 4dbb55ad..00000000 --- a/lib/text/urls-processor.js +++ /dev/null @@ -1,75 +0,0 @@ -var EscapeStore = require('./escape-store'); -var reduceUrls = require('../urls/reduce'); - -var lineBreak = require('os').EOL; - -function UrlsProcessor(context, saveWaypoints, keepUrlQuotes) { - this.urls = new EscapeStore('URL'); - this.context = context; - this.saveWaypoints = saveWaypoints; - this.keepUrlQuotes = keepUrlQuotes; -} - -// Strip urls by replacing them by a special -// marker for further restoring. It's done via string scanning -// instead of regexps to speed up the process. -UrlsProcessor.prototype.escape = function (data) { - var breaksCount; - var lastBreakAt; - var indent; - var saveWaypoints = this.saveWaypoints; - var self = this; - - return reduceUrls(data, this.context, function (url, tempData) { - if (saveWaypoints) { - breaksCount = url.split(lineBreak).length - 1; - lastBreakAt = url.lastIndexOf(lineBreak); - indent = lastBreakAt > 0 ? - url.substring(lastBreakAt + lineBreak.length).length : - url.length; - } - - var placeholder = self.urls.store(url, saveWaypoints ? [breaksCount, indent] : null); - tempData.push(placeholder); - }); -}; - -function normalize(url, keepUrlQuotes) { - url = url - .replace(/^url/gi, 'url') - .replace(/\\?\n|\\?\r\n/g, '') - .replace(/(\s{2,}|\s)/g, ' ') - .replace(/^url\((['"])? /, 'url($1') - .replace(/ (['"])?\)$/, '$1)'); - - if (/url\(".*'.*"\)/.test(url) || /url\('.*".*'\)/.test(url)) - return url; - - if (!keepUrlQuotes && !/^['"].+['"]$/.test(url) && !/url\(.*[\s\(\)].*\)/.test(url) && !/url\(['"]data:[^;]+;charset/.test(url)) - url = url.replace(/["']/g, ''); - - return url; -} - -UrlsProcessor.prototype.restore = function (data) { - var tempData = []; - var cursor = 0; - - for (; cursor < data.length;) { - var nextMatch = this.urls.nextMatch(data, cursor); - if (nextMatch.start < 0) - break; - - tempData.push(data.substring(cursor, nextMatch.start)); - var url = normalize(this.urls.restore(nextMatch.match), this.keepUrlQuotes); - tempData.push(url); - - cursor = nextMatch.end; - } - - return tempData.length > 0 ? - tempData.join('') + data.substring(cursor, data.length) : - data; -}; - -module.exports = UrlsProcessor; diff --git a/lib/tokenizer/extract-properties.js b/lib/tokenizer/extract-properties.js deleted file mode 100644 index 9a55eb5d..00000000 --- a/lib/tokenizer/extract-properties.js +++ /dev/null @@ -1,193 +0,0 @@ -var split = require('../utils/split'); - -var COMMA = ','; -var FORWARD_SLASH = '/'; - -var AT_RULE = 'at-rule'; - -var IMPORTANT_WORD = 'important'; -var IMPORTANT_TOKEN = '!'+IMPORTANT_WORD; -var IMPORTANT_WORD_MATCH = new RegExp('^'+IMPORTANT_WORD+'$', 'i'); -var IMPORTANT_TOKEN_MATCH = new RegExp('^'+IMPORTANT_TOKEN+'$', 'i'); - -function selectorName(value) { - return value[0]; -} - -function noop() {} - -function withoutComments(string, into, heading, context) { - var matcher = heading ? /^__ESCAPED_COMMENT_/ : /__ESCAPED_COMMENT_/; - var track = heading ? context.track : noop; // don't track when comment not in a heading as we do it later in `trackComments` - - while (matcher.test(string)) { - var startOfComment = string.indexOf('__'); - var endOfComment = string.indexOf('__', startOfComment + 1) + 2; - var comment = string.substring(startOfComment, endOfComment); - string = string.substring(0, startOfComment) + string.substring(endOfComment); - - track(comment); - into.push(comment); - } - - return string; -} - -function withoutHeadingComments(string, into, context) { - return withoutComments(string, into, true, context); -} - -function withoutInnerComments(string, into, context) { - return withoutComments(string, into, false, context); -} - -function trackComments(comments, into, context) { - for (var i = 0, l = comments.length; i < l; i++) { - context.track(comments[i]); - into.push(comments[i]); - } -} - -function extractProperties(string, selectors, context) { - var list = []; - var innerComments = []; - var valueSeparator = /[ ,\/]/; - - if (typeof string != 'string') - return []; - - if (string.indexOf(')') > -1) - string = string.replace(/\)([^\s_;:,\)])/g, context.sourceMap ? ') __ESCAPED_COMMENT_CLEAN_CSS(0,-1)__ $1' : ') $1'); - - if (string.indexOf('ESCAPED_URL_CLEAN_CSS') > -1) - string = string.replace(/(ESCAPED_URL_CLEAN_CSS[^_]+?__)/g, context.sourceMap ? '$1 __ESCAPED_COMMENT_CLEAN_CSS(0,-1)__ ' : '$1 '); - - var candidates = split(string, ';', false, '{', '}'); - - for (var i = 0, l = candidates.length; i < l; i++) { - var candidate = candidates[i]; - var firstColonAt = candidate.indexOf(':'); - - var atRule = candidate.trim()[0] == '@'; - if (atRule) { - context.track(candidate); - list.push([AT_RULE, candidate.trim()]); - continue; - } - - if (firstColonAt == -1) { - context.track(candidate); - if (candidate.indexOf('__ESCAPED_COMMENT_SPECIAL') > -1) - list.push(candidate.trim()); - continue; - } - - if (candidate.indexOf('{') > 0 && candidate.indexOf('{') < firstColonAt) { - context.track(candidate); - continue; - } - - var body = []; - var name = candidate.substring(0, firstColonAt); - - innerComments = []; - - if (name.indexOf('__ESCAPED_COMMENT') > -1) - name = withoutHeadingComments(name, list, context); - - if (name.indexOf('__ESCAPED_COMMENT') > -1) - name = withoutInnerComments(name, innerComments, context); - - body.push([name.trim()].concat(context.track(name, true))); - context.track(':'); - - trackComments(innerComments, list, context); - - var firstBraceAt = candidate.indexOf('{'); - var isVariable = name.trim().indexOf('--') === 0; - if (isVariable && firstBraceAt > 0) { - var blockPrefix = candidate.substring(firstColonAt + 1, firstBraceAt + 1); - var blockSuffix = candidate.substring(candidate.indexOf('}')); - var blockContent = candidate.substring(firstBraceAt + 1, candidate.length - blockSuffix.length); - - context.track(blockPrefix); - body.push(extractProperties(blockContent, selectors, context)); - list.push(body); - context.track(blockSuffix); - context.track(i < l - 1 ? ';' : ''); - - continue; - } - - var values = split(candidate.substring(firstColonAt + 1), valueSeparator, true); - - if (values.length == 1 && values[0] === '') { - context.warnings.push('Empty property \'' + name + '\' inside \'' + selectors.filter(selectorName).join(',') + '\' selector. Ignoring.'); - continue; - } - - for (var j = 0, m = values.length; j < m; j++) { - var value = values[j]; - var trimmed = value.trim(); - - if (trimmed.length === 0) - continue; - - var lastCharacter = trimmed[trimmed.length - 1]; - var endsWithNonSpaceSeparator = trimmed.length > 1 && (lastCharacter == COMMA || lastCharacter == FORWARD_SLASH); - - if (endsWithNonSpaceSeparator) - trimmed = trimmed.substring(0, trimmed.length - 1); - - if (trimmed.indexOf('__ESCAPED_COMMENT_CLEAN_CSS(0,-') > -1) { - context.track(trimmed); - continue; - } - - innerComments = []; - - if (trimmed.indexOf('__ESCAPED_COMMENT') > -1) - trimmed = withoutHeadingComments(trimmed, list, context); - - if (trimmed.indexOf('__ESCAPED_COMMENT') > -1) - trimmed = withoutInnerComments(trimmed, innerComments, context); - - if (trimmed.length === 0) { - trackComments(innerComments, list, context); - continue; - } - - var pos = body.length - 1; - if (IMPORTANT_WORD_MATCH.test(trimmed) && body[pos][0] == '!') { - context.track(trimmed); - body[pos - 1][0] += IMPORTANT_TOKEN; - body.pop(); - continue; - } - - if (IMPORTANT_TOKEN_MATCH.test(trimmed) || (IMPORTANT_WORD_MATCH.test(trimmed) && body[pos][0][body[pos][0].length - 1] == '!')) { - context.track(trimmed); - body[pos][0] += trimmed; - continue; - } - - body.push([trimmed].concat(context.track(value, true))); - - trackComments(innerComments, list, context); - - if (endsWithNonSpaceSeparator) { - body.push([lastCharacter]); - context.track(lastCharacter); - } - } - - if (i < l - 1) - context.track(';'); - - list.push(body); - } - - return list; -} - -module.exports = extractProperties; diff --git a/lib/tokenizer/extract-selectors.js b/lib/tokenizer/extract-selectors.js deleted file mode 100644 index dc8a47fa..00000000 --- a/lib/tokenizer/extract-selectors.js +++ /dev/null @@ -1,17 +0,0 @@ -var split = require('../utils/split'); - -function extractSelectors(string, context) { - var list = []; - var metadata; - var selectors = split(string, ','); - - for (var i = 0, l = selectors.length; i < l; i++) { - metadata = context.track(selectors[i], true, i); - context.track(','); - list.push([selectors[i].trim()].concat(metadata)); - } - - return list; -} - -module.exports = extractSelectors; diff --git a/lib/tokenizer/marker.js b/lib/tokenizer/marker.js new file mode 100644 index 00000000..1129c6e1 --- /dev/null +++ b/lib/tokenizer/marker.js @@ -0,0 +1,22 @@ +var Marker = { + AT: '@', + BACK_SLASH: '\\', + CLOSE_BRACE: '}', + CLOSE_ROUND_BRACKET: ')', + COLON: ':', + COMMA: ',', + DOUBLE_QUOTE: '"', + EXCLAMATION: '!', + FORWARD_SLASH: '/', + NEW_LINE_NIX: '\n', + NEW_LINE_WIN: '\r', + OPEN_BRACE: '{', + OPEN_ROUND_BRACKET: '(', + SEMICOLON: ';', + SINGLE_QUOTE: '\'', + SPACE: ' ', + STAR: '*', + UNDERSCORE: '_' +}; + +module.exports = Marker; diff --git a/lib/tokenizer/token.js b/lib/tokenizer/token.js new file mode 100644 index 00000000..06a1a136 --- /dev/null +++ b/lib/tokenizer/token.js @@ -0,0 +1,13 @@ +var Token = { + AT_RULE: 'at-rule', // e.g. `@import`, `@charset` + AT_RULE_BLOCK: 'at-rule-block', // e.g. `@font-face` + BLOCK: 'block', // e.g. `@media`, `@keyframes` + COMMENT: 'comment', // e.g. `/* comment */` + PROPERTY: 'property', // e.g. `color:red` + PROPERTY_BLOCK: 'property-block', // e.g. `--var:{color:red}` + PROPERTY_NAME: 'property-name', // e.g. `color` + PROPERTY_VALUE: 'property-value', // e.g. `red` + RULE: 'rule' // e.g `div > a` +}; + +module.exports = Token; diff --git a/lib/tokenizer/tokenize.js b/lib/tokenizer/tokenize.js index b262c494..ed9b73ae 100644 --- a/lib/tokenizer/tokenize.js +++ b/lib/tokenizer/tokenize.js @@ -1,268 +1,401 @@ -var extractProperties = require('./extract-properties'); -var extractSelectors = require('./extract-selectors'); -var track = require('../source-maps/track'); -var split = require('../utils/split'); - -var path = require('path'); - -var flatBlock = /(@(font\-face|page|\-ms\-viewport|\-o\-viewport|viewport|counter\-style)|\\@.+?)/; - -function tokenize(data, outerContext) { - var chunks = split(normalize(data), '}', true, '{', '}'); - if (chunks.length === 0) - return []; - - var context = { - chunk: chunks.shift(), - chunks: chunks, - column: 0, - cursor: 0, - line: 1, - mode: 'top', - resolvePath: outerContext.options.explicitTarget ? - relativePathResolver(outerContext.options.root, outerContext.options.target) : - null, - source: undefined, - sourceMap: outerContext.options.sourceMap, - sourceMapInlineSources: outerContext.options.sourceMapInlineSources, - sourceMapTracker: outerContext.inputSourceMapTracker, - sourceReader: outerContext.sourceReader, - sourceTracker: outerContext.sourceTracker, - state: [], - track: outerContext.options.sourceMap ? - function (data, snapshotMetadata, fallbacks) { return [[track(data, context, snapshotMetadata, fallbacks)]]; } : - function () { return []; }, - warnings: outerContext.warnings +var Marker = require('./marker'); +var Token = require('./token'); + +var Level = { + BLOCK: 'block', + COMMENT: 'comment', + DOUBLE_QUOTE: 'double-quote', + RULE: 'rule', + SINGLE_QUOTE: 'single-quote' +}; + +var AT_RULES = [ + '@charset', + '@import' +]; + +var BLOCK_RULES = [ + '@-moz-document', + '@document', + '@-moz-keyframes', + '@-ms-keyframes', + '@-o-keyframes', + '@-webkit-keyframes', + '@keyframes', + '@media', + '@supports' +]; + +var TAIL_BROKEN_VALUE_PATTERN = /[\s|\}]*$/; + +function tokenize(source, externalContext) { + var internalContext = { + level: Level.BLOCK, + position: { + source: externalContext.source || undefined, + line: 1, + column: 0, + index: 0 + } }; - return intoTokens(context); -} - -function normalize(data) { - return data.replace(/\r\n/g, '\n'); -} - -function relativePathResolver(root, target) { - var rebaseTo = path.relative(root, target); - - return function (relativeTo, sourcePath) { - return relativeTo != sourcePath ? - path.normalize(path.join(path.relative(rebaseTo, path.dirname(relativeTo)), sourcePath)) : - sourcePath; - }; + return intoTokens(source, externalContext, internalContext, false); } -function whatsNext(context) { - var mode = context.mode; - var chunk = context.chunk; - var closest; +function intoTokens(source, externalContext, internalContext, isNested) { + var allTokens = []; + var newTokens = allTokens; + var lastToken; + var ruleToken; + var ruleTokens = []; + var propertyToken; + var metadata; + var metadatas = []; + var level = internalContext.level; + var levels = []; + var buffer = []; + var buffers = []; + var roundBracketLevel = 0; + var isQuoted; + var isSpace; + var isNewLineNix; + var isNewLineWin; + var isCommentStart; + var wasCommentStart = false; + var isCommentEnd; + var wasCommentEnd = false; + var isEscaped; + var seekingValue = false; + var position = internalContext.position; + + for (; position.index < source.length; position.index++) { + var character = source[position.index]; + + isQuoted = level == Level.SINGLE_QUOTE || level == Level.DOUBLE_QUOTE; + isSpace = character == Marker.SPACE; + isNewLineNix = character == Marker.NEW_LINE_NIX; + isNewLineWin = character == Marker.NEW_LINE_NIX && source[position.index - 1] == Marker.NEW_LINE_WIN; + isCommentStart = !wasCommentEnd && level != Level.COMMENT && !isQuoted && character == Marker.STAR && source[position.index - 1] == Marker.FORWARD_SLASH; + isCommentEnd = !wasCommentStart && level == Level.COMMENT && character == Marker.FORWARD_SLASH && source[position.index - 1] == Marker.STAR; + + metadata = buffer.length === 0 ? + metadataFrom(position, 0, externalContext) : + metadata; + + if (isEscaped) { + // previous character was a backslash + buffer.push(character); + } else if (!isCommentEnd && level == Level.COMMENT) { + buffer.push(character); + } else if (isCommentStart && (level == Level.BLOCK || level == Level.RULE) && buffer.length > 1) { + // comment start within block preceded by some content, e.g. div/*<-- + metadatas.push(metadata); + buffer.push(character); + buffers.push(buffer.slice(0, buffer.length - 2)); + + buffer = buffer.slice(buffer.length - 2); + metadata = metadataFrom(position, 1, externalContext); + + levels.push(level); + level = Level.COMMENT; + } else if (isCommentStart) { + // comment start, e.g. /*<-- + levels.push(level); + level = Level.COMMENT; + buffer.push(character); + } else if (isCommentEnd) { + // comment end, e.g. /* comment */<-- + lastToken = [Token.COMMENT, [buffer.join('').trim() + character, [metadata]]]; + newTokens.push(lastToken); + + level = levels.pop(); + metadata = metadatas.pop() || null; + buffer = buffers.pop() || []; + } else if (character == Marker.SINGLE_QUOTE && !isQuoted) { + // single quotation start, e.g. a[href^='https<-- + levels.push(level); + level = Level.SINGLE_QUOTE; + buffer.push(character); + } else if (character == Marker.SINGLE_QUOTE && level == Level.SINGLE_QUOTE) { + // single quotation end, e.g. a[href^='https'<-- + level = levels.pop(); + buffer.push(character); + } else if (character == Marker.DOUBLE_QUOTE && !isQuoted) { + // double quotation start, e.g. a[href^="<-- + levels.push(level); + level = Level.DOUBLE_QUOTE; + buffer.push(character); + } else if (character == Marker.DOUBLE_QUOTE && level == Level.DOUBLE_QUOTE) { + // double quotation end, e.g. a[href^="https"<-- + level = levels.pop(); + buffer.push(character); + } else if (!isCommentStart && !isCommentEnd && character != Marker.CLOSE_ROUND_BRACKET && character != Marker.OPEN_ROUND_BRACKET && level != Level.COMMENT && !isQuoted && roundBracketLevel > 0) { + // character inside any function, e.g. hsla(.<-- + buffer.push(character); + } else if (character == Marker.OPEN_ROUND_BRACKET && !isQuoted && level != Level.COMMENT && !seekingValue) { + // round open bracket, e.g. @import url(<-- + buffer.push(character); + + roundBracketLevel++; + } else if (character == Marker.CLOSE_ROUND_BRACKET && !isQuoted && level != Level.COMMENT && !seekingValue) { + // round open bracket, e.g. @import url(test.css)<-- + buffer.push(character); + + roundBracketLevel--; + } else if (character == Marker.SEMICOLON && level == Level.BLOCK) { + // semicolon ending rule at block level, e.g. @import '...';<-- + allTokens.push([Token.AT_RULE, buffer.join('').trim(), [metadata]]); + + buffer = []; + } else if (character == Marker.COMMA && level == Level.BLOCK && ruleToken) { + // comma separator at block level, e.g. a,div,<-- + ruleToken[1].push([buffer.join('').trim(), [metadata]]); + + buffer = []; + } else if (character == Marker.COMMA && level == Level.BLOCK && tokenTypeFrom(buffer) == Token.AT_RULE) { + // comma separator at block level, e.g. @import url(...) screen,<-- + // keep iterating as end semicolon will create the token + buffer.push(character); + } else if (character == Marker.COMMA && level == Level.BLOCK) { + // comma separator at block level, e.g. a,<-- + ruleToken = [tokenTypeFrom(buffer), [[buffer.join('').trim(), [metadata]]], []]; + + buffer = []; + } else if (character == Marker.OPEN_BRACE && level == Level.BLOCK && ruleToken && ruleToken[0] == Token.BLOCK) { + // open brace opening at-rule at block level, e.g. @media{<-- + ruleToken[1].push([buffer.join('').trim(), [metadata]]); + allTokens.push(ruleToken); + + levels.push(level); + position.column++; + position.index++; + buffer = []; + + ruleToken[2] = intoTokens(source, externalContext, internalContext, true); + ruleToken = null; + } else if (character == Marker.OPEN_BRACE && level == Level.BLOCK && tokenTypeFrom(buffer) == Token.BLOCK) { + // open brace opening at-rule at block level, e.g. @media{<-- + ruleToken = ruleToken || [Token.BLOCK, [], []]; + ruleToken[1].push([buffer.join('').trim(), [metadata]]); + allTokens.push(ruleToken); + + levels.push(level); + position.column++; + position.index++; + buffer = []; + + ruleToken[2] = intoTokens(source, externalContext, internalContext, true); + ruleToken = null; + } else if (character == Marker.OPEN_BRACE && level == Level.BLOCK) { + // open brace opening rule at block level, e.g. div{<-- + ruleToken = ruleToken || [tokenTypeFrom(buffer), [], []]; + ruleToken[1].push([buffer.join('').trim(), [metadata]]); + newTokens = ruleToken[2]; + allTokens.push(ruleToken); + + levels.push(level); + level = Level.RULE; + buffer = []; + } else if (character == Marker.OPEN_BRACE && level == Level.RULE && seekingValue) { + // open brace opening rule at rule level, e.g. div{--variable:{<-- + ruleTokens.push(ruleToken); + ruleToken = [Token.PROPERTY_BLOCK, []]; + propertyToken.push(ruleToken); + newTokens = ruleToken[1]; + + levels.push(level); + level = Level.RULE; + seekingValue = false; + } else if (character == Marker.COLON && level == Level.RULE && !seekingValue) { + // colon at rule level, e.g. a{color:<-- + propertyToken = [Token.PROPERTY, [Token.PROPERTY_NAME, buffer.join('').trim(), [metadata]]]; + newTokens.push(propertyToken); + + seekingValue = true; + buffer = []; + } else if (character == Marker.SEMICOLON && level == Level.RULE && propertyToken && ruleTokens.length > 0 && buffer.length > 0 && buffer[0] == Marker.AT) { + // semicolon at rule level for at-rule, e.g. a{--color:{@apply(--other-color);<-- + ruleToken[1].push([Token.AT_RULE, buffer.join('').trim(), [metadata]]); + + buffer = []; + } else if (character == Marker.SEMICOLON && level == Level.RULE && propertyToken && buffer.length > 0) { + // semicolon at rule level, e.g. a{color:red;<-- + propertyToken.push([Token.PROPERTY_VALUE, buffer.join('').trim(), [metadata]]); + + seekingValue = false; + buffer = []; + } else if (character == Marker.SEMICOLON && level == Level.RULE && propertyToken && buffer.length === 0) { + // semicolon after bracketed value at rule level, e.g. a{color:rgb(...);<-- + seekingValue = false; + } else if (character == Marker.SEMICOLON && level == Level.RULE && buffer.length > 0 && buffer[0] == Marker.AT) { + // semicolon for at-rule at rule level, e.g. a{@apply(--variable);<-- + newTokens.push([Token.AT_RULE, buffer.join('').trim(), [metadata]]); + + seekingValue = false; + buffer = []; + } else if (character == Marker.SEMICOLON && level == Level.RULE && buffer.length === 0) { + // stray semicolon at rule level, e.g. a{;<-- + // noop + } else if (character == Marker.CLOSE_BRACE && level == Level.RULE && propertyToken && seekingValue && buffer.length > 0 && ruleTokens.length > 0) { + // close brace at rule level, e.g. a{--color:{color:red}<-- + propertyToken.push([Token.PROPERTY_VALUE, buffer.join(''), [metadata]]); + propertyToken = null; + ruleToken = ruleTokens.pop(); + newTokens = ruleToken[2]; + + level = levels.pop(); + seekingValue = false; + buffer = []; + } else if (character == Marker.CLOSE_BRACE && level == Level.RULE && propertyToken && buffer.length > 0 && buffer[0] == Marker.AT && ruleTokens.length > 0) { + // close brace at rule level for at-rule, e.g. a{--color:{@apply(--other-color)}<-- + ruleToken[1].push([Token.AT_RULE, buffer.join(''), [metadata]]); + propertyToken = null; + ruleToken = ruleTokens.pop(); + newTokens = ruleToken[2]; + + level = levels.pop(); + seekingValue = false; + buffer = []; + } else if (character == Marker.CLOSE_BRACE && level == Level.RULE && propertyToken && ruleTokens.length > 0) { + // close brace at rule level after space, e.g. a{--color:{color:red }<-- + propertyToken = null; + ruleToken = ruleTokens.pop(); + newTokens = ruleToken[2]; + + level = levels.pop(); + seekingValue = false; + } else if (character == Marker.CLOSE_BRACE && level == Level.RULE && propertyToken && buffer.length > 0) { + // close brace at rule level, e.g. a{color:red}<-- + propertyToken.push([Token.PROPERTY_VALUE, buffer.join(''), [metadata]]); + propertyToken = null; + ruleToken = ruleTokens.pop(); + newTokens = allTokens; + + level = levels.pop(); + seekingValue = false; + buffer = []; + } else if (character == Marker.CLOSE_BRACE && level == Level.RULE && buffer.length > 0 && buffer[0] == Marker.AT) { + // close brace after at-rule at rule level, e.g. a{@apply(--variable)}<-- + ruleToken = null; + newTokens.push([Token.AT_RULE, buffer.join('').trim(), [metadata]]); + newTokens = allTokens; + + level = levels.pop(); + seekingValue = false; + buffer = []; + } else if (character == Marker.CLOSE_BRACE && level == Level.RULE) { + // close brace after at-rule at rule level, e.g. a{color:red;}<-- + ruleToken = null; + newTokens = allTokens; + + level = levels.pop(); + seekingValue = false; + } else if (character == Marker.CLOSE_BRACE && level == Level.BLOCK && !isNested && position.index < source.length - 1) { + // stray close brace at block level, e.g. @media screen {...}}<-- + externalContext.warnings.push('Extra \'}\' at line ' + position.line + ', column ' + position.column); + // noop + } else if (character == Marker.CLOSE_BRACE && level == Level.BLOCK) { + // close brace at block level, e.g. @media screen {...}<-- + break; + } else if (character == Marker.OPEN_ROUND_BRACKET && level == Level.RULE && seekingValue) { + // round open bracket, e.g. a{color:hsla(<-- + buffer.push(character); + roundBracketLevel++; + } else if (character == Marker.CLOSE_ROUND_BRACKET && level == Level.RULE && seekingValue && roundBracketLevel == 1) { + // round close bracket, e.g. a{color:hsla(0,0%,0%)<-- + buffer.push(character); + propertyToken.push([Token.PROPERTY_VALUE, buffer.join('').trim(), [metadata]]); + + roundBracketLevel--; + buffer = []; + } else if (character == Marker.CLOSE_ROUND_BRACKET && level == Level.RULE && seekingValue) { + // round close bracket within other brackets, e.g. a{width:calc((10rem / 2)<-- + buffer.push(character); + roundBracketLevel--; + } else if (character == Marker.FORWARD_SLASH && source[position.index + 1] != Marker.STAR && level == Level.RULE && seekingValue && buffer.length > 0) { + // forward slash within a property, e.g. a{background:url(image.png) 0 0/<-- + propertyToken.push([Token.PROPERTY_VALUE, buffer.join('').trim(), [metadata]]); + propertyToken.push([Token.PROPERTY_VALUE, character, [metadataFrom(position, 0, externalContext)]]); + + buffer = []; + } else if (character == Marker.FORWARD_SLASH && source[position.index + 1] != Marker.STAR && level == Level.RULE && seekingValue) { + // forward slash within a property after space, e.g. a{background:url(image.png) 0 0 /<-- + propertyToken.push([Token.PROPERTY_VALUE, character, [metadataFrom(position, 0, externalContext)]]); + + buffer = []; + } else if (character == Marker.COMMA && level == Level.RULE && seekingValue && buffer.length > 0) { + // comma within a property, e.g. a{background:url(image.png),<-- + propertyToken.push([Token.PROPERTY_VALUE, buffer.join('').trim(), [metadata]]); + propertyToken.push([Token.PROPERTY_VALUE, character, [metadataFrom(position, 0, externalContext)]]); + + buffer = []; + } else if (character == Marker.COMMA && level == Level.RULE && seekingValue) { + // comma within a property after space, e.g. a{background:url(image.png) ,<-- + propertyToken.push([Token.PROPERTY_VALUE, character, [metadataFrom(position, 0, externalContext)]]); + + buffer = []; + } else if ((isSpace || (isNewLineNix && !isNewLineWin)) && level == Level.RULE && seekingValue && propertyToken && buffer.length > 0) { + // space or *nix newline within property, e.g. a{margin:0 <-- + propertyToken.push([Token.PROPERTY_VALUE, buffer.join('').trim(), [metadata]]); + + buffer = []; + } else if (isNewLineWin && level == Level.RULE && seekingValue && propertyToken && buffer.length > 1) { + // win newline within property, e.g. a{margin:0\r\n<-- + propertyToken.push([Token.PROPERTY_VALUE, buffer.join('').trim(), [metadata]]); + + buffer = []; + } else if (isNewLineWin && level == Level.RULE && seekingValue) { + // win newline + buffer = []; + } else if (buffer.length == 1 && isNewLineWin) { + // ignore windows newline which is composed of two characters + buffer.pop(); + } else if (buffer.length > 0 || !isSpace && !isNewLineNix && !isNewLineWin) { + // any character + buffer.push(character); + } - if (chunk.length == context.cursor) { - if (context.chunks.length === 0) - return null; + isEscaped = character == Marker.BACK_SLASH; + wasCommentStart = isCommentStart; + wasCommentEnd = isCommentEnd; - context.chunk = chunk = context.chunks.shift(); - context.cursor = 0; + position.line = (isNewLineWin || isNewLineNix) ? position.line + 1 : position.line; + position.column = (isNewLineWin || isNewLineNix) ? 0 : position.column + 1; } - if (mode == 'body') { - if (chunk[context.cursor] == '}') - return [context.cursor, 'bodyEnd']; - - if (chunk.indexOf('}', context.cursor) == -1) - return null; - - closest = context.cursor + split(chunk.substring(context.cursor - 1), '}', true, '{', '}')[0].length - 2; - return [closest, 'bodyEnd']; + if (seekingValue) { + externalContext.warnings.push('Missing \'}\' at line ' + position.line + ', column ' + position.column); } - var nextSpecial = chunk.indexOf('@', context.cursor); - var nextEscape = chunk.indexOf('__ESCAPED_', context.cursor); - var nextBodyStart = chunk.indexOf('{', context.cursor); - var nextBodyEnd = chunk.indexOf('}', context.cursor); - - if (nextSpecial > -1 && context.cursor > 0 && !/\s|\{|\}|\/|_|,|;/.test(chunk.substring(nextSpecial - 1, nextSpecial))) { - nextSpecial = -1; + if (seekingValue && buffer.length > 0) { + propertyToken.push([Token.PROPERTY_VALUE, buffer.join('').replace(TAIL_BROKEN_VALUE_PATTERN, ''), [metadata]]); } - if (nextEscape > -1 && /\S/.test(chunk.substring(context.cursor, nextEscape))) - nextEscape = -1; - - closest = nextSpecial; - if (closest == -1 || (nextEscape > -1 && nextEscape < closest)) - closest = nextEscape; - if (closest == -1 || (nextBodyStart > -1 && nextBodyStart < closest)) - closest = nextBodyStart; - if (closest == -1 || (nextBodyEnd > -1 && nextBodyEnd < closest)) - closest = nextBodyEnd; - - if (closest == -1) - return; - if (nextEscape === closest) - return [closest, 'escape']; - if (nextBodyStart === closest) - return [closest, 'bodyStart']; - if (nextBodyEnd === closest) - return [closest, 'bodyEnd']; - if (nextSpecial === closest) - return [closest, 'special']; + return allTokens; } -function intoTokens(context) { - var chunk = context.chunk; - var tokenized = []; - var newToken; - var value; - - while (true) { - var next = whatsNext(context); - if (!next) { - var whatsLeft = context.chunk.substring(context.cursor); - if (whatsLeft.trim().length > 0) { - if (context.mode == 'body') { - context.warnings.push('Missing \'}\' after \'' + whatsLeft + '\'. Ignoring.'); - } else { - tokenized.push(['text', [whatsLeft]]); - } - context.cursor += whatsLeft.length; - } - break; - } - - var nextSpecial = next[0]; - var what = next[1]; - var nextEnd; - var oldMode; +function metadataFrom(position, columnDelta, externalContext) { + var metadata = [position.line, position.column - columnDelta, position.source]; + metadata = externalContext.inputSourceMap ? + externalContext.inputSourceMapTracker.originalPositionFor(metadata) : + metadata; - chunk = context.chunk; - - if (context.cursor != nextSpecial && what != 'bodyEnd') { - var spacing = chunk.substring(context.cursor, nextSpecial); - var leadingWhitespace = /^\s+/.exec(spacing); - - if (leadingWhitespace) { - context.cursor += leadingWhitespace[0].length; - context.track(leadingWhitespace[0]); - } - } - - if (what == 'special') { - var firstOpenBraceAt = chunk.indexOf('{', nextSpecial); - var firstSemicolonAt = chunk.indexOf(';', nextSpecial); - var isSingle = firstSemicolonAt > -1 && (firstOpenBraceAt == -1 || firstSemicolonAt < firstOpenBraceAt); - var isBroken = firstOpenBraceAt == -1 && firstSemicolonAt == -1; - if (isBroken) { - context.warnings.push('Broken declaration: \'' + chunk.substring(context.cursor) + '\'.'); - context.cursor = chunk.length; - } else if (isSingle) { - nextEnd = chunk.indexOf(';', nextSpecial + 1); - value = chunk.substring(context.cursor, nextEnd + 1); - - tokenized.push([ - 'at-rule', - [value].concat(context.track(value, true)) - ]); - - context.track(';'); - context.cursor = nextEnd + 1; - } else { - nextEnd = chunk.indexOf('{', nextSpecial + 1); - value = chunk.substring(context.cursor, nextEnd); - - var trimmedValue = value.trim(); - var isFlat = flatBlock.test(trimmedValue); - oldMode = context.mode; - context.cursor = nextEnd + 1; - context.mode = isFlat ? 'body' : 'block'; - - newToken = [ - isFlat ? 'flat-block' : 'block' - ]; - - newToken.push([trimmedValue].concat(context.track(value, true))); - context.track('{'); - newToken.push(intoTokens(context)); - - if (typeof newToken[2] == 'string') - newToken[2] = extractProperties(newToken[2], [[trimmedValue]], context); - - context.mode = oldMode; - context.track('}'); - - tokenized.push(newToken); - } - } else if (what == 'escape') { - nextEnd = chunk.indexOf('__', nextSpecial + 1); - var escaped = chunk.substring(context.cursor, nextEnd + 2); - var isStartSourceMarker = !!context.sourceTracker.nextStart(escaped); - var isEndSourceMarker = !!context.sourceTracker.nextEnd(escaped); - - if (isStartSourceMarker) { - context.track(escaped); - context.state.push({ - source: context.source, - line: context.line, - column: context.column - }); - context.source = context.sourceTracker.nextStart(escaped).filename; - context.line = 1; - context.column = 0; - } else if (isEndSourceMarker) { - var oldState = context.state.pop(); - context.source = oldState.source; - context.line = oldState.line; - context.column = oldState.column; - context.track(escaped); - } else { - if (escaped.indexOf('__ESCAPED_COMMENT_SPECIAL') === 0) - tokenized.push(['text', [escaped]]); - - context.track(escaped); - } - - context.cursor = nextEnd + 2; - } else if (what == 'bodyStart') { - var selectors = extractSelectors(chunk.substring(context.cursor, nextSpecial), context); - - oldMode = context.mode; - context.cursor = nextSpecial + 1; - context.mode = 'body'; - - var body = extractProperties(intoTokens(context), selectors, context); - - context.track('{'); - context.mode = oldMode; - - tokenized.push([ - 'selector', - selectors, - body - ]); - } else if (what == 'bodyEnd') { - // extra closing brace at the top level can be safely ignored - if (context.mode == 'top') { - var at = context.cursor; - var warning = chunk[context.cursor] == '}' ? - 'Unexpected \'}\' in \'' + chunk.substring(at - 20, at + 20) + '\'. Ignoring.' : - 'Unexpected content: \'' + chunk.substring(at, nextSpecial + 1) + '\'. Ignoring.'; - - context.warnings.push(warning); - context.cursor = nextSpecial + 1; - continue; - } - - if (context.mode == 'block') - context.track(chunk.substring(context.cursor, nextSpecial)); - if (context.mode != 'block') - tokenized = chunk.substring(context.cursor, nextSpecial); - - context.cursor = nextSpecial + 1; + return metadata; +} - break; - } +function tokenTypeFrom(buffer) { + var isAtRule = buffer[0] == Marker.AT || buffer[0] == Marker.UNDERSCORE; + var ruleWord = buffer.join('').split(/\s/)[0]; + + if (isAtRule && BLOCK_RULES.indexOf(ruleWord) > -1) { + return Token.BLOCK; + } else if (isAtRule && AT_RULES.indexOf(ruleWord) > -1) { + return Token.AT_RULE; + } else if (isAtRule) { + return Token.AT_RULE_BLOCK; + } else { + return Token.RULE; } - - return tokenized; } module.exports = tokenize; diff --git a/lib/utils/input-source-map-tracker-2.js b/lib/utils/input-source-map-tracker-2.js new file mode 100644 index 00000000..ff379482 --- /dev/null +++ b/lib/utils/input-source-map-tracker-2.js @@ -0,0 +1,30 @@ +var SourceMapConsumer = require('source-map').SourceMapConsumer; + +function inputSourceMapTracker() { + var maps = {}; + + return { + originalPositionFor: originalPositionFor.bind(null, maps), + track: track.bind(null, maps) + }; +} + +function originalPositionFor(maps, metadata) { + var line = metadata[0]; + var column = metadata[1]; + var source = metadata[2]; + + return source in maps ? + toMetadata(maps[source].originalPositionFor({ line: line, column: column })) : + metadata; +} + +function toMetadata(asHash) { + return [asHash.line, asHash.column, asHash.source]; +} + +function track(maps, source, data) { + maps[source] = new SourceMapConsumer(data); +} + +module.exports = inputSourceMapTracker; diff --git a/test/text/comments-processor-test.js b/test/text/comments-processor-test.js deleted file mode 100644 index 2b92e3c7..00000000 --- a/test/text/comments-processor-test.js +++ /dev/null @@ -1,192 +0,0 @@ -var vows = require('vows'); -var assert = require('assert'); -var CommentsProcessor = require('../../lib/text/comments-processor'); - -var lineBreak = require('os').EOL; -var otherLineBreak = lineBreak == '\n' ? '\r\n' : '\n'; - -function processorContext(name, context, keepSpecialComments, keepBreaks, saveWaypoints) { - var vowContext = {}; - - function escaped (expected) { - return function (source) { - var escaped = new CommentsProcessor(null, keepSpecialComments, keepBreaks, saveWaypoints).escape(source); - assert.equal(escaped, expected); - }; - } - - function restored (expected) { - return function (source) { - var processor = new CommentsProcessor(null, keepSpecialComments, keepBreaks, saveWaypoints); - var restored = processor.restore(processor.escape(source)); - assert.equal(restored, expected); - }; - } - - for (var key in context) { - vowContext[name + ' - ' + key] = { - topic: context[key][0], - escaped: escaped(context[key][1]), - restored: restored(context[key][2]) - }; - } - - return vowContext; -} - -vows.describe(CommentsProcessor) - .addBatch( - processorContext('all', { - 'no comments': [ - 'a{color:red}', - 'a{color:red}', - 'a{color:red}' - ], - 'one comment': [ - '/* some text */', - '', - '' - ], - 'one special comment': [ - '/*! some text */', - '__ESCAPED_COMMENT_SPECIAL_CLEAN_CSS0__', - '/*! some text */' - ], - 'two comments': [ - '/* one text *//* another text */', - '', - '' - ], - 'two same comments': [ - '/* one text *//* one text */', - '', - '' - ], - 'two special comments': [ - '/*! one text *//*! another text */', - '__ESCAPED_COMMENT_SPECIAL_CLEAN_CSS0____ESCAPED_COMMENT_SPECIAL_CLEAN_CSS1__', - '/*! one text *//*! another text */' - ], - 'two special comments when second with @ sign and preceded by quote sign': [ - '/*! comment 1 */*{box-sizing: border-box}div:before{content:" "}/*! comment 2 */div{display:inline-block}', - '__ESCAPED_COMMENT_SPECIAL_CLEAN_CSS0__*{box-sizing: border-box}div:before{content:" "}__ESCAPED_COMMENT_SPECIAL_CLEAN_CSS1__div{display:inline-block}', - '/*! comment 1 */*{box-sizing: border-box}div:before{content:" "}/*! comment 2 */div{display:inline-block}' - ], - 'commented selector': [ - '/* a{color:red} */', - '', - '' - ], - 'quoted comment': [ - 'a{content:"/* text */"}', - 'a{content:"/* text */"}', - 'a{content:"/* text */"}' - ] - }, '*') - ) - .addBatch( - processorContext('one', { - 'one comment': [ - '/* some text */', - '', - '' - ], - 'one special comment': [ - '/*! some text */', - '__ESCAPED_COMMENT_SPECIAL_CLEAN_CSS0__', - '/*! some text */' - ], - 'two special comments': [ - '/*! one text *//*! another text */', - '__ESCAPED_COMMENT_SPECIAL_CLEAN_CSS0____ESCAPED_COMMENT_SPECIAL_CLEAN_CSS1__', - '/*! one text */' - ] - }, '1') - ) - .addBatch( - processorContext('zero', { - 'one comment': [ - '/* some text */', - '', - '' - ], - 'one special comment': [ - '/*! some text */', - '__ESCAPED_COMMENT_SPECIAL_CLEAN_CSS0__', - '' - ], - 'two special comments': [ - '/*! one text *//*! another text */', - '__ESCAPED_COMMENT_SPECIAL_CLEAN_CSS0____ESCAPED_COMMENT_SPECIAL_CLEAN_CSS1__', - '' - ] - }, '0') - ) - .addBatch( - processorContext('zero with breaks', { - 'remove comment and a break': [ - 'a{}' + lineBreak + '/*! some text */' + lineBreak + 'p{}', - 'a{}' + lineBreak + '__ESCAPED_COMMENT_SPECIAL_CLEAN_CSS0__' + lineBreak + 'p{}', - 'a{}' + lineBreak + 'p{}' - ] - }, '0', true) - ) - .addBatch( - processorContext('one with breaks', { - 'keeps content as is': [ - 'a{}/*! some text */p{}', - 'a{}__ESCAPED_COMMENT_SPECIAL_CLEAN_CSS0__p{}', - 'a{}/*! some text */p{}' - ], - 'keeps if not given already': [ - 'a{}/*! some text */' + lineBreak + 'p{}', - 'a{}__ESCAPED_COMMENT_SPECIAL_CLEAN_CSS0__' + lineBreak + 'p{}', - 'a{}/*! some text */' + lineBreak + 'p{}' - ], - 'if given an other platform break already': [ - 'a{}/*! some text */' + otherLineBreak + 'p{}', - 'a{}__ESCAPED_COMMENT_SPECIAL_CLEAN_CSS0__' + otherLineBreak + 'p{}', - 'a{}/*! some text */' + otherLineBreak + 'p{}' - ] - }, '1', true) - ) - .addBatch( - processorContext('waypoints', { - 'one comment': [ - '/* some text */', - '__ESCAPED_COMMENT_CLEAN_CSS0(0,15)__', - '' - ], - 'one special comment': [ - '/*! some text */', - '__ESCAPED_COMMENT_SPECIAL_CLEAN_CSS0(0,16)__', - '/*! some text */' - ], - 'two comments': [ - '/* one text *//* another text */', - '__ESCAPED_COMMENT_CLEAN_CSS0(0,14)____ESCAPED_COMMENT_CLEAN_CSS1(0,33)__', - '' - ], - 'two same comments': [ - '/* one text *//* one text */', - '__ESCAPED_COMMENT_CLEAN_CSS0(0,14)____ESCAPED_COMMENT_CLEAN_CSS0(0,29)__', - '' - ], - 'two special comments': [ - '/*! one text *//*! another text */', - '__ESCAPED_COMMENT_SPECIAL_CLEAN_CSS0(0,15)____ESCAPED_COMMENT_SPECIAL_CLEAN_CSS1(0,35)__', - '/*! one text *//*! another text */' - ], - 'two special comments with line breaks': [ - '/*! one text' + lineBreak + '*//*! another' + lineBreak + ' text' + lineBreak + ' */', - '__ESCAPED_COMMENT_SPECIAL_CLEAN_CSS0(1,2)____ESCAPED_COMMENT_SPECIAL_CLEAN_CSS1(2,3)__', - '/*! one text' + lineBreak + '*//*! another' + lineBreak + ' text' + lineBreak + ' */' - ], - 'three special comments with line breaks and content in between': [ - '/*! one text' + lineBreak + '*/a{}/*! ' + lineBreak + 'test */p{color:red}/*! another' + lineBreak + ' text' + lineBreak + lineBreak + ' */', - '__ESCAPED_COMMENT_SPECIAL_CLEAN_CSS0(1,2)__a{}__ESCAPED_COMMENT_SPECIAL_CLEAN_CSS1(1,7)__p{color:red}__ESCAPED_COMMENT_SPECIAL_CLEAN_CSS2(3,4)__', - '/*! one text' + lineBreak + '*/a{}/*! ' + lineBreak + 'test */p{color:red}/*! another' + lineBreak + ' text' + lineBreak + lineBreak + ' */' - ] - }, '*', false, true) - ) - .export(module); diff --git a/test/text/escape-store-test.js b/test/text/escape-store-test.js deleted file mode 100644 index 9a27ecd9..00000000 --- a/test/text/escape-store-test.js +++ /dev/null @@ -1,69 +0,0 @@ -var vows = require('vows'); -var assert = require('assert'); -var EscapeStore = require('../../lib/text/escape-store'); - -vows.describe(EscapeStore) - .addBatch({ - 'no metadata': { - topic: function () { - return new EscapeStore('TEST'); - }, - store: function (escapeStore) { - var placeholder = escapeStore.store('data'); - assert.equal(placeholder, '__ESCAPED_TEST_CLEAN_CSS0__'); - }, - match: function (escapeStore) { - var next = escapeStore.nextMatch('prefix__ESCAPED_TEST_CLEAN_CSS0__suffix'); - assert.equal(next.start, 6); - assert.equal(next.end, 33); - assert.equal(next.match, '__ESCAPED_TEST_CLEAN_CSS0__'); - }, - restore: function (escapeStore) { - var data = escapeStore.restore('__ESCAPED_TEST_CLEAN_CSS0__'); - assert.equal(data, 'data'); - } - }, - 'with metadata': { - topic: function () { - return new EscapeStore('TEST'); - }, - store: function (escapeStore) { - var placeholder = escapeStore.store('data', ['brown', 'fox', 'jumped', 'over']); - assert.equal(placeholder, '__ESCAPED_TEST_CLEAN_CSS0(brown,fox,jumped,over)__'); - }, - match: function (escapeStore) { - var next = escapeStore.nextMatch('prefix__ESCAPED_TEST_CLEAN_CSS0(brown,fox,jumped,over)__suffix'); - assert.equal(next.start, 6); - assert.equal(next.end, 56); - assert.equal(next.match, '__ESCAPED_TEST_CLEAN_CSS0(brown,fox,jumped,over)__'); - }, - restore: function (escapeStore) { - var data = escapeStore.restore('__ESCAPED_TEST_CLEAN_CSS0(brown,fox,jumped,over)__'); - assert.equal(data, 'data'); - } - }, - 'same data with different metadata': { - topic: function () { - return new EscapeStore('TEST'); - }, - 'store first': function (escapeStore) { - escapeStore.store('data1', [0, 1, 2]); - var placeholder = escapeStore.store('data1', [1, 2, 3]); - - assert.equal(placeholder, '__ESCAPED_TEST_CLEAN_CSS0(1,2,3)__'); - } - }, - 'with different metadata but same index': { - topic: function () { - var escapeStore = new EscapeStore('TEST'); - escapeStore.store('data', ['brown', 'fox']); - escapeStore.store('data', ['jumped', 'over']); - return escapeStore; - }, - restore: function (escapeStore) { - var data = escapeStore.restore('__ESCAPED_TEST_CLEAN_CSS0(a,lazy,dog)__'); - assert.equal(data, 'data'); - } - } - }) - .export(module); diff --git a/test/text/expressions-processor-test.js b/test/text/expressions-processor-test.js deleted file mode 100644 index 97760009..00000000 --- a/test/text/expressions-processor-test.js +++ /dev/null @@ -1,105 +0,0 @@ -var vows = require('vows'); -var assert = require('assert'); -var ExpressionsProcessor = require('../../lib/text/expressions-processor'); - -var lineBreak = require('os').EOL; - -function processorContext(name, context, saveWaypoints) { - var vowContext = {}; - - function escaped (expected) { - return function (source) { - var escaped = new ExpressionsProcessor(saveWaypoints).escape(source); - assert.equal(escaped, expected); - }; - } - - function restored (expected) { - return function (source) { - var processor = new ExpressionsProcessor(saveWaypoints); - var restored = processor.restore(processor.escape(source)); - assert.equal(restored, expected); - }; - } - - for (var key in context) { - vowContext[name + ' - ' + key] = { - topic: context[key][0], - escaped: escaped(context[key][1]), - restored: restored(context[key][2]) - }; - } - - return vowContext; -} - -vows.describe(ExpressionsProcessor) - .addBatch( - processorContext('basic', { - 'empty': [ - 'a{color:expression()}', - 'a{color:__ESCAPED_EXPRESSION_CLEAN_CSS0__}', - 'a{color:expression()}' - ], - 'method call': [ - 'a{color:expression(this.parentNode.currentStyle.color)}', - 'a{color:__ESCAPED_EXPRESSION_CLEAN_CSS0__}', - 'a{color:expression(this.parentNode.currentStyle.color)}' - ], - 'multiple calls': [ - 'a{color:expression(x = 0 , this.parentNode.currentStyle.color)}', - 'a{color:__ESCAPED_EXPRESSION_CLEAN_CSS0__}', - 'a{color:expression(x = 0 , this.parentNode.currentStyle.color)}' - ], - 'mixed content': [ - 'a{zoom:expression(this.runtimeStyle[\"zoom\"] = \'1\', this.innerHTML = \'\')}', - 'a{zoom:__ESCAPED_EXPRESSION_CLEAN_CSS0__}', - 'a{zoom:expression(this.runtimeStyle[\"zoom\"] = \'1\', this.innerHTML = \'\')}' - ], - 'complex': [ - 'a{width:expression((this.parentNode.innerWidth + this.parentNode.innerHeight) / 2 )}', - 'a{width:__ESCAPED_EXPRESSION_CLEAN_CSS0__}', - 'a{width:expression((this.parentNode.innerWidth + this.parentNode.innerHeight) / 2 )}' - ], - 'with parentheses': [ - 'a{width:expression(this.parentNode.innerText == \')\' ? \'5px\' : \'10px\' )}', - 'a{width:__ESCAPED_EXPRESSION_CLEAN_CSS0__}', - 'a{width:expression(this.parentNode.innerText == \')\' ? \'5px\' : \'10px\' )}' - ], - 'open ended (broken)': [ - 'a{width:expression(this.parentNode.innerText == }', - 'a{width:__ESCAPED_EXPRESSION_CLEAN_CSS0__}', - 'a{width:expression(this.parentNode.innerText == }' - ], - 'function call & advanced': [ - 'a{zoom:expression(function (el){el.style.zoom="1"}(this))}', - 'a{zoom:__ESCAPED_EXPRESSION_CLEAN_CSS0__}', - 'a{zoom:expression(function (el){el.style.zoom="1"}(this))}' - ], - 'with more properties': [ - 'a{color:red;zoom:expression(function (el){el.style.zoom="1"}(this));display:block}', - 'a{color:red;zoom:__ESCAPED_EXPRESSION_CLEAN_CSS0__;display:block}', - 'a{color:red;zoom:expression(function (el){el.style.zoom="1"}(this));display:block}' - ] - }) - ) - .addBatch( - processorContext('waypoints', { - 'empty': [ - 'a{color:expression()}', - 'a{color:__ESCAPED_EXPRESSION_CLEAN_CSS0(0,12)__}', - 'a{color:expression()}' - ], - 'method call': [ - 'a{color:expression(this.parentNode.currentStyle.color)}', - 'a{color:__ESCAPED_EXPRESSION_CLEAN_CSS0(0,46)__}', - 'a{color:expression(this.parentNode.currentStyle.color)}' - ], - 'line break call': [ - 'a{color:expression(' + lineBreak + 'this.parentNode.currentStyle.color)}', - 'a{color:__ESCAPED_EXPRESSION_CLEAN_CSS0(1,35)__}', - 'a{color:expression(' + lineBreak + 'this.parentNode.currentStyle.color)}' - ] - }, true) - ) - .export(module); diff --git a/test/text/free-text-processor-test.js b/test/text/free-text-processor-test.js deleted file mode 100644 index 8c8c880f..00000000 --- a/test/text/free-text-processor-test.js +++ /dev/null @@ -1,100 +0,0 @@ -var vows = require('vows'); -var assert = require('assert'); -var FreeTextProcessor = require('../../lib/text/free-text-processor'); - -var lineBreak = require('os').EOL; - -function processorContext(name, context, saveWaypoints) { - var vowContext = {}; - - function escaped (expected) { - return function (source) { - var escaped = new FreeTextProcessor(saveWaypoints).escape(source); - assert.equal(escaped, expected); - }; - } - - function restored (expected) { - return function (source) { - var processor = new FreeTextProcessor(saveWaypoints); - var restored = processor.restore(processor.escape(source)); - assert.equal(restored, expected); - }; - } - - for (var key in context) { - vowContext[name + ' - ' + key] = { - topic: context[key][0], - escaped: escaped(context[key][1]), - restored: restored(context[key][2]) - }; - } - - return vowContext; -} - -vows.describe(FreeTextProcessor) - .addBatch( - processorContext('basic', { - 'no quotes': [ - 'a{color:red;display:block}', - 'a{color:red;display:block}', - 'a{color:red;display:block}' - ], - 'single quoted': [ - 'a{color:red;content:\'1234\';display:block}', - 'a{color:red;content:__ESCAPED_FREE_TEXT_CLEAN_CSS0__;display:block}', - 'a{color:red;content:\'1234\';display:block}' - ], - 'double quoted': [ - 'a{color:red;content:"1234";display:block}', - 'a{color:red;content:__ESCAPED_FREE_TEXT_CLEAN_CSS0__;display:block}', - 'a{color:red;content:"1234";display:block}' - ], - 'inside format': [ - '@font-face{font-family:X;src:X.ttf format(\'opentype\')}', - '@font-face{font-family:X;src:X.ttf format(__ESCAPED_FREE_TEXT_CLEAN_CSS0__)}', - '@font-face{font-family:X;src:X.ttf format(\'opentype\')}' - ], - 'inside local': [ - '@font-face{font-family:X;src:local(\'Pacifico\') format(\'opentype\')}', - '@font-face{font-family:X;src:local(__ESCAPED_FREE_TEXT_CLEAN_CSS0__) format(__ESCAPED_FREE_TEXT_CLEAN_CSS1__)}', - '@font-face{font-family:X;src:local(\'Pacifico\') format(\'opentype\')}' - ], - 'attribute': [ - 'a[data-type="search"]{}', - 'a[data-type=__ESCAPED_FREE_TEXT_CLEAN_CSS0__]{}', - 'a[data-type=search]{}' - ], - 'font name': [ - 'a{font-family:"Times","Times New Roman",serif}', - 'a{font-family:__ESCAPED_FREE_TEXT_CLEAN_CSS0__,__ESCAPED_FREE_TEXT_CLEAN_CSS1__,serif}', - 'a{font-family:Times,"Times New Roman",serif}' - ] - }) - ) - .addBatch( - processorContext('waypoints', { - 'no quotes': [ - 'a{color:red;display:block}', - 'a{color:red;display:block}', - 'a{color:red;display:block}' - ], - 'single quoted': [ - 'a{color:red;content:\'1234\';display:block}', - 'a{color:red;content:__ESCAPED_FREE_TEXT_CLEAN_CSS0(0,6)__;display:block}', - 'a{color:red;content:\'1234\';display:block}' - ], - 'double quoted': [ - 'a{color:red;content:"1234";display:block}', - 'a{color:red;content:__ESCAPED_FREE_TEXT_CLEAN_CSS0(0,6)__;display:block}', - 'a{color:red;content:"1234";display:block}' - ], - 'with breaks': [ - 'a{color:red;content:"1234' + lineBreak + '56";display:block}', - 'a{color:red;content:__ESCAPED_FREE_TEXT_CLEAN_CSS0(1,3)__;display:block}', - 'a{color:red;content:"1234' + lineBreak + '56";display:block}' - ] - }, true) - ) - .export(module); diff --git a/test/text/urls-processor-test.js b/test/text/urls-processor-test.js deleted file mode 100644 index daa74e35..00000000 --- a/test/text/urls-processor-test.js +++ /dev/null @@ -1,145 +0,0 @@ -var vows = require('vows'); -var assert = require('assert'); -var UrlsProcessor = require('../../lib/text/urls-processor'); - -var lineBreak = require('os').EOL; - -function processorContext(name, context, saveWaypoints) { - var vowContext = {}; - - function escaped (expected) { - return function (source) { - var escaped = new UrlsProcessor(null, saveWaypoints).escape(source); - assert.equal(escaped, expected); - }; - } - - function restored (expected) { - return function (source) { - var processor = new UrlsProcessor(null, saveWaypoints); - var restored = processor.restore(processor.escape(source)); - assert.equal(restored, expected); - }; - } - - for (var key in context) { - vowContext[name + ' - ' + key] = { - topic: context[key][0], - escaped: escaped(context[key][1]), - restored: restored(context[key][2]) - }; - } - - return vowContext; -} - -vows.describe(UrlsProcessor) - .addBatch( - processorContext('basic', { - 'no urls': [ - 'a{color:red}', - 'a{color:red}', - 'a{color:red}' - ], - 'unquoted': [ - 'div{background:url(some/file.png) repeat}', - 'div{background:__ESCAPED_URL_CLEAN_CSS0__ repeat}', - 'div{background:url(some/file.png) repeat}' - ], - 'single quoted': [ - 'div{background:url(\'some/file.png\') repeat}', - 'div{background:__ESCAPED_URL_CLEAN_CSS0__ repeat}', - 'div{background:url(some/file.png) repeat}' - ], - 'single quoted with whitespace': [ - 'div{background:url(\'some/file name.png\') repeat}', - 'div{background:__ESCAPED_URL_CLEAN_CSS0__ repeat}', - 'div{background:url(\'some/file name.png\') repeat}' - ], - 'double quoted': [ - 'div{background:url("some/file.png") repeat}', - 'div{background:__ESCAPED_URL_CLEAN_CSS0__ repeat}', - 'div{background:url(some/file.png) repeat}' - ], - 'double quoted with whitespace': [ - 'div{background:url("some/file name.png") repeat}', - 'div{background:__ESCAPED_URL_CLEAN_CSS0__ repeat}', - 'div{background:url("some/file name.png") repeat}' - ], - 'multiple': [ - 'div{background:url(one/file.png) repeat}p{background:url(second/file.png)}', - 'div{background:__ESCAPED_URL_CLEAN_CSS0__ repeat}p{background:__ESCAPED_URL_CLEAN_CSS1__}', - 'div{background:url(one/file.png) repeat}p{background:url(second/file.png)}' - ], - 'whitespace': [ - 'div{background:url(\' some/\nfile.png \') repeat}', - 'div{background:__ESCAPED_URL_CLEAN_CSS0__ repeat}', - 'div{background:url(some/file.png) repeat}' - ], - 'unescaped closing brackets': [ - 'div{background:url("some/).png") repeat}', - 'div{background:__ESCAPED_URL_CLEAN_CSS0__ repeat}', - 'div{background:url("some/).png") repeat}' - ], - 'followed by attribute matcher selector': [ - 'a{background:url(url)}div:not([test]){color:red}', - 'a{background:__ESCAPED_URL_CLEAN_CSS0__}div:not([test]){color:red}', - 'a{background:url(url)}div:not([test]){color:red}' - ], - 'data URI wrapped in single quotes': [ - 'a{background-image:url(\'data:image/svg+xml;charset=utf-8,\')}', - 'a{background-image:__ESCAPED_URL_CLEAN_CSS0__}', - 'a{background-image:url(\'data:image/svg+xml;charset=utf-8,\')}' - ], - 'data URI wrapped in double quotes': [ - 'a{background-image:url("data:image/svg+xml;charset=utf-8,")}', - 'a{background-image:__ESCAPED_URL_CLEAN_CSS0__}', - 'a{background-image:url("data:image/svg+xml;charset=utf-8,")}' - ], - 'two quoted data URIs with closing brackets': [ - '.a{cursor:url("data:application/octet-stream;base64,A...rotate(30 60,60)...="),move!important}.b{cursor:url("data:application/octet-stream;base64,A...rotate(30 60,60)...=")}', - '.a{cursor:__ESCAPED_URL_CLEAN_CSS0__,move!important}.b{cursor:__ESCAPED_URL_CLEAN_CSS0__}', - '.a{cursor:url("data:application/octet-stream;base64,A...rotate(30 60,60)...="),move!important}.b{cursor:url("data:application/octet-stream;base64,A...rotate(30 60,60)...=")}', - ], - 'quoted data URI with unbalanced closing brackets': [ - '.a{cursor:url("data:application/octet-stream;base64,A...%3C!--)--%3E...=");color:red;}', - '.a{cursor:__ESCAPED_URL_CLEAN_CSS0__;color:red;}', - '.a{cursor:url("data:application/octet-stream;base64,A...%3C!--)--%3E...=");color:red;}' - ] - }) - ) - .addBatch( - processorContext('waypoints', { - 'no urls': [ - 'a{color:red}', - 'a{color:red}', - 'a{color:red}' - ], - 'unquoted': [ - 'div{background:url(some/file.png) repeat}', - 'div{background:__ESCAPED_URL_CLEAN_CSS0(0,18)__ repeat}', - 'div{background:url(some/file.png) repeat}' - ], - 'single quoted': [ - 'div{background:url(\'some/file.png\') repeat}', - 'div{background:__ESCAPED_URL_CLEAN_CSS0(0,20)__ repeat}', - 'div{background:url(some/file.png) repeat}' - ], - 'double quoted': [ - 'div{background:url("some/file.png") repeat}', - 'div{background:__ESCAPED_URL_CLEAN_CSS0(0,20)__ repeat}', - 'div{background:url(some/file.png) repeat}' - ], - 'with line breaks': [ - 'div{background:url("' + lineBreak + 'some/' + lineBreak + 'file.png") repeat}', - 'div{background:__ESCAPED_URL_CLEAN_CSS0(2,10)__ repeat}', - 'div{background:url(some/file.png) repeat}' - ], - 'multiple': [ - 'div{background:url(one/file.png) repeat}p{background:url(second/file.png)}', - 'div{background:__ESCAPED_URL_CLEAN_CSS0(0,17)__ repeat}p{background:__ESCAPED_URL_CLEAN_CSS1(0,20)__}', - 'div{background:url(one/file.png) repeat}p{background:url(second/file.png)}' - ] - }, true) - ) - .export(module); diff --git a/test/tokenizer/tokenize-test.js b/test/tokenizer/tokenize-test.js new file mode 100644 index 00000000..e8a11f44 --- /dev/null +++ b/test/tokenizer/tokenize-test.js @@ -0,0 +1,3540 @@ +var vows = require('vows'); +var assert = require('assert'); +var tokenize = require('../../lib/tokenizer/tokenize'); +var inputSourceMapTracker = require('../../lib/utils/input-source-map-tracker-2'); + +var fs = require('fs'); +var path = require('path'); +var inputMapPath = path.join('test', 'fixtures', 'source-maps', 'styles.css.map'); +var inputMap = fs.readFileSync(inputMapPath, 'utf-8'); + +function tokenizerContext(group, specs) { + var ctx = {}; + + function tokenizedContext(target) { + return function (tokenized) { + assert.deepEqual(tokenized, target); + }; + } + + function toTokens(source) { + return function () { + return tokenize(source, { + options: {}, + warnings: [] + }); + }; + } + + for (var test in specs) { + var target = specs[test][1]; + + ctx[group + ' ' + test] = { + topic: toTokens(specs[test][0]), + tokenized: tokenizedContext(target) + }; + } + + return ctx; +} + +vows.describe(tokenize) + .addBatch( + tokenizerContext('basic', { + 'no content': [ + '', + [] + ], + 'a comment': [ + '/* comment */', + [ + [ + 'comment', + [ + '/* comment */', + [ + [1, 0, undefined] + ] + ] + ] + ] + ], + 'a comment followed by a break': [ + '/* comment */\n', + [ + [ + 'comment', + [ + '/* comment */', + [ + [1, 0, undefined] + ] + ] + ] + ] + ], + 'a comment with forward slash as first character': [ + '/*/ comment */', + [ + [ + 'comment', + [ + '/*/ comment */', + [ + [1, 0, undefined] + ] + ] + ] + ] + ], + 'a rule between two comments': [ + '/* comment 1 */*/* comment 2 */{}', + [ + [ + 'comment', + [ + '/* comment 1 */', + [ + [1, 0, undefined] + ] + ] + ], + [ + 'comment', + [ + '/* comment 2 */', + [ + [1, 16, undefined] + ] + ] + ], + [ + 'rule', + [ + [ + '*', + [ + [1, 15, undefined] + ] + ] + ], + [] + ] + ] + ], + 'an empty rule': [ + 'a{}', + [ + [ + 'rule', + [ + [ + 'a', + [ + [1, 0, undefined] + ] + ] + ], + [] + ] + ] + ], + 'a comment with breaks followed by an empty rule': [ + '/* comment \n\n */a{}', + [ + [ + 'comment', + [ + '/* comment \n\n */', + [ + [1, 0, undefined] + ] + ] + ], + [ + 'rule', + [ + [ + 'a', + [ + [3, 3, undefined] + ] + ] + ], + [] + ] + ] + ], + 'an empty rule with whitespace': [ + 'a{ \n }', + [ + [ + 'rule', + [ + [ + 'a', + [ + [1, 0, undefined] + ] + ] + ], + [] + ] + ] + ], + 'a rule with a property': [ + 'a{color:red}', + [ + [ + 'rule', + [ + [ + 'a', + [ + [1, 0, undefined] + ] + ] + ], + [ + [ + 'property', + [ + 'property-name', + 'color', + [ + [1, 2, undefined] + ] + ], + [ + 'property-value', + 'red', + [ + [1, 8, undefined] + ] + ] + ] + ] + ] + ] + ], + 'a rule with a multi value property': [ + 'a{margin:0px 2px 1px}', + [ + [ + 'rule', + [ + [ + 'a', + [ + [1, 0, undefined] + ] + ] + ], + [ + [ + 'property', + [ + 'property-name', + 'margin', + [ + [1, 2, undefined] + ] + ], + [ + 'property-value', + '0px', + [ + [1, 9, undefined] + ] + ], + [ + 'property-value', + '2px', + [ + [1, 13, undefined] + ] + ], + [ + 'property-value', + '1px', + [ + [1, 17, undefined] + ] + ] + ] + ] + ] + ] + ], + 'a rule with a filter property': [ + 'a{filter:progid:DXImageTransform.Microsoft.Alpha(Opacity=80)}', + [ + [ + 'rule', + [ + [ + 'a', + [ + [1, 0, undefined] + ] + ] + ], + [ + [ + 'property', + [ + 'property-name', + 'filter', + [ + [1, 2, undefined] + ] + ], + [ + 'property-value', + 'progid:DXImageTransform.Microsoft.Alpha(Opacity=80)', + [ + [1, 9, undefined] + ] + ] + ] + ] + ] + ] + ], + 'a rule with whitespace': [ + 'a {color:red;\n\ndisplay :\r\n block }', + [ + [ + 'rule', + [ + [ + 'a', + [ + [1, 0, undefined] + ] + ] + ], + [ + [ + 'property', + [ + 'property-name', + 'color', + [ + [1, 3, undefined] + ] + ], + [ + 'property-value', + 'red', + [ + [1, 9, undefined] + ] + ] + ], + [ + 'property', + [ + 'property-name', + 'display', + [ + [3, 0, undefined] + ] + ], + [ + 'property-value', + 'block', + [ + [4, 2, undefined] + ] + ] + ] + ] + ] + ] + ], + 'a rule with suffix whitespace': [ + 'div a{color:red\r\n}', + [ + [ + 'rule', + [ + [ + 'div a', + [ + [1, 0, undefined] + ] + ] + ], + [ + [ + 'property', + [ + 'property-name', + 'color', + [ + [1, 6, undefined] + ] + ], + [ + 'property-value', + 'red', + [ + [1, 12, undefined] + ] + ] + ] + ] + ] + ] + ], + 'a rule with whitespace in functions': [ + 'a{color:rgba( 255, 255, 0, 0.5 )}', + [ + [ + 'rule', + [ + [ + 'a', + [ + [1, 0, undefined] + ] + ] + ], + [ + [ + 'property', + [ + 'property-name', + 'color', + [ + [1, 2, undefined] + ] + ], + [ + 'property-value', + 'rgba( 255, 255, 0, 0.5 )', + [ + [1, 8, undefined] + ] + ] + ] + ] + ] + ] + ], + 'a rule with functions and no whitespace breaks': [ + 'a{background:rgba(255,255,0,0.5)url(test.png)repeat no-repeat}', + [ + [ + 'rule', + [ + [ + 'a', + [ + [1, 0, undefined] + ] + ] + ], + [ + [ + 'property', + [ + 'property-name', + 'background', + [ + [1, 2, undefined] + ] + ], + [ + 'property-value', + 'rgba(255,255,0,0.5)', + [ + [1, 13, undefined] + ] + ], + [ + 'property-value', + 'url(test.png)', + [ + [1, 32, undefined] + ] + ], + [ + 'property-value', + 'repeat', + [ + [1, 45, undefined] + ] + ], + [ + 'property-value', + 'no-repeat', + [ + [1, 52, undefined] + ] + ] + ] + ] + ] + ] + ], + 'a rule with url and no whitespace breaks': [ + 'a{background:url(image.png)50px/25%}', + [ + [ + 'rule', + [ + [ + 'a', + [ + [1, 0, undefined] + ] + ] + ], + [ + [ + 'property', + [ + 'property-name', + 'background', + [ + [1, 2, undefined] + ] + ], + [ + 'property-value', + 'url(image.png)', + [ + [1, 13, undefined] + ] + ], + [ + 'property-value', + '50px', + [ + [1, 27, undefined] + ] + ], + [ + 'property-value', + '/', + [ + [1, 31, undefined] + ] + ], + [ + 'property-value', + '25%', + [ + [1, 32, undefined] + ] + ] + ] + ] + ] + ] + ], + 'a rule with two properties where first ends with a round close bracket': [ + 'a{width:calc(100% - 25px);width:50rem}', + [ + [ + 'rule', + [ + [ + 'a', + [ + [1, 0, undefined] + ] + ] + ], + [ + [ + 'property', + [ + 'property-name', + 'width', + [ + [1, 2, undefined] + ] + ], + [ + 'property-value', + 'calc(100% - 25px)', + [ + [1, 8, undefined] + ] + ] + ], + [ + 'property', + [ + 'property-name', + 'width', + [ + [1, 26, undefined] + ] + ], + [ + 'property-value', + '50rem', + [ + [1, 32, undefined] + ] + ] + ] + ] + ] + ] + ], + 'a rule with empty properties': [ + 'a{color:red; ; ; ;}', + [ + [ + 'rule', + [ + [ + 'a', + [ + [1, 0, undefined] + ] + ] + ], + [ + [ + 'property', + [ + 'property-name', + 'color', + [ + [1, 2, undefined] + ] + ], + [ + 'property-value', + 'red', + [ + [1, 8, undefined] + ] + ] + ] + ] + ] + ] + ], + 'a rule with quoted attribute': [ + 'a[data-kind="one two"]{color:red}', + [ + [ + 'rule', + [ + [ + 'a[data-kind="one two"]', + [ + [1, 0, undefined] + ] + ] + ], + [ + [ + 'property', + [ + 'property-name', + 'color', + [ + [1, 23, undefined] + ] + ], + [ + 'property-value', + 'red', + [ + [1, 29, undefined] + ] + ] + ] + ] + ] + ] + ], + 'a rule with escaped quote': [ + '.this-class\\\'s-got-an-apostrophe{color:red}', + [ + [ + 'rule', + [ + [ + '.this-class\\\'s-got-an-apostrophe', + [ + [1, 0, undefined] + ] + ] + ], + [ + [ + 'property', + [ + 'property-name', + 'color', + [ + [1, 33, undefined] + ] + ], + [ + 'property-value', + 'red', + [ + [1, 39, undefined] + ] + ] + ] + ] + ] + ] + ], + 'a rule with quoted comment': [ + 'a{background:url(\'/* this is silly */\')}', + [ + [ + 'rule', + [ + [ + 'a', + [ + [1, 0, undefined] + ] + ] + ], + [ + [ + 'property', + [ + 'property-name', + 'background', + [ + [1, 2, undefined] + ] + ], + [ + 'property-value', + 'url(\'/* this is silly */\')', + [ + [1, 13, undefined] + ] + ] + ] + ] + ] + ] + ], + 'a rule with quote and comments inside quote': [ + 'a{content:\'"abc /* 1 */"\'}', + [ + [ + 'rule', + [ + [ + 'a', + [ + [1, 0, undefined] + ] + ] + ], + [ + [ + 'property', + [ + 'property-name', + 'content', + [ + [1, 2, undefined] + ] + ], + [ + 'property-value', + '\'"abc /* 1 */"\'', + [ + [1, 10, undefined] + ] + ] + ] + ] + ] + ] + ], + 'a double rule': [ + 'a,\n\ndiv.class > p {color:red}', + [ + [ + 'rule', + [ + [ + 'a', + [ + [1, 0, undefined] + ] + ], + [ + 'div.class > p', + [ + [3, 0, undefined] + ] + ] + ], + [ + [ + 'property', + [ + 'property-name', + 'color', + [ + [3, 15, undefined] + ] + ], + [ + 'property-value', + 'red', + [ + [3, 21, undefined] + ] + ] + ] + ] + ] + ] + ], + 'a triple rule': [ + 'b,a,div{}', + [ + [ + 'rule', + [ + [ + 'b', + [ + [1, 0, undefined] + ] + ], + [ + 'a', + [ + [1, 2, undefined] + ] + ], + [ + 'div', + [ + [1, 4, undefined] + ] + ] + ], + [] + ] + ] + ], + 'two rules': [ + 'a{color:red}\n div{color:blue}', + [ + [ + 'rule', + [ + [ + 'a', + [ + [1, 0, undefined] + ] + ] + ], + [ + [ + 'property', + [ + 'property-name', + 'color', + [ + [1, 2, undefined] + ] + ], + [ + 'property-value', + 'red', + [ + [1, 8, undefined] + ] + ] + ] + ] + ], + [ + 'rule', + [ + [ + 'div', + [ + [2, 1, undefined] + ] + ] + ], + [ + [ + 'property', + [ + 'property-name', + 'color', + [ + [2, 5, undefined] + ] + ], + [ + 'property-value', + 'blue', + [ + [2, 11, undefined] + ] + ] + ] + ] + ] + ] + ], + 'two comments and a rule separated by newline': [ + '/* comment 1 */\n/* comment 2 */\ndiv{}', + [ + [ + 'comment', + [ + '/* comment 1 */', + [ + [1, 0, undefined] + ] + ] + ], + [ + 'comment', + [ + '/* comment 2 */', + [ + [2, 0, undefined] + ] + ] + ], + [ + 'rule', + [ + [ + 'div', + [ + [3, 0, undefined] + ] + ] + ], + [] + ] + ] + ], + 'rule wrapped between comments': [ + '/* comment 1 */div/* comment 2 */{color:red}', + [ + [ + 'comment', + [ + '/* comment 1 */', + [ + [1, 0, undefined] + ] + ] + ], + [ + 'comment', + [ + '/* comment 2 */', + [ + [1, 18, undefined] + ] + ] + ], + [ + 'rule', + [ + [ + 'div', + [ + [1, 15, undefined] + ] + ] + ], + [ + [ + 'property', + [ + 'property-name', + 'color', + [ + [1, 34, undefined] + ] + ], + [ + 'property-value', + 'red', + [ + [1, 40, undefined] + ] + ] + ] + ] + ] + ] + ], + 'two properties wrapped between comments': [ + 'div{/* comment 1 */color:red/* comment 2 */}', + [ + [ + 'rule', + [ + [ + 'div', + [ + [1, 0, undefined] + ] + ] + ], + [ + [ + 'comment', + [ + '/* comment 1 */', + [ + [1, 4, undefined] + ] + ] + ], + [ + 'property', + [ + 'property-name', + 'color', + [ + [1, 19, undefined] + ] + ], + [ + 'property-value', + 'red', + [ + [1, 25, undefined] + ] + ] + ], + [ + 'comment', + [ + '/* comment 2 */', + [ + [1, 28, undefined] + ] + ] + ] + ] + ] + ] + ], + 'multiple values wrapped between comments #1': [ + 'div{background:url(image.png),/* comment */red}', + [ + [ + 'rule', + [ + [ + 'div', + [ + [1, 0, undefined] + ] + ] + ], + [ + [ + 'property', + [ + 'property-name', + 'background', + [ + [1, 4, undefined] + ] + ], + [ + 'property-value', + 'url(image.png)', + [ + [1, 15, undefined] + ] + ], + [ + 'property-value', + ',', + [ + [1, 29, undefined] + ] + ], + [ + 'property-value', + 'red', + [ + [1, 43, undefined] + ] + ] + ], + [ + 'comment', + [ + '/* comment */', + [ + [1, 30, undefined] + ] + ] + ] + ] + ] + ] + ], + 'multiple values wrapped between comments #2': [ + 'div{background:url(image.png),red/* comment */}', + [ + [ + 'rule', + [ + [ + 'div', + [ + [1, 0, undefined] + ] + ] + ], + [ + [ + 'property', + [ + 'property-name', + 'background', + [ + [1, 4, undefined] + ] + ], + [ + 'property-value', + 'url(image.png)', + [ + [1, 15, undefined] + ] + ], + [ + 'property-value', + ',', + [ + [1, 29, undefined] + ] + ], + [ + 'property-value', + 'red', + [ + [1, 30, undefined] + ] + ] + ], + [ + 'comment', + [ + '/* comment */', + [ + [1, 33, undefined] + ] + ] + ] + ] + ] + ] + ], + 'multiple values wrapped between comments #3': [ + 'div{background:url(image.png),rgba(0,0,0,/* comment */0.1)}', + [ + [ + 'rule', + [ + [ + 'div', + [ + [1, 0, undefined] + ] + ] + ], + [ + [ + 'property', + [ + 'property-name', + 'background', + [ + [1, 4, undefined] + ] + ], + [ + 'property-value', + 'url(image.png)', + [ + [1, 15, undefined] + ] + ], + [ + 'property-value', + ',', + [ + [1, 29, undefined] + ] + ], + [ + 'property-value', + 'rgba(0,0,0,0.1)', + [ + [1, 30, undefined] + ] + ] + ], + [ + 'comment', + [ + '/* comment */', + [ + [1, 41, undefined] + ] + ] + ], + ] + ] + ] + ], + 'pseudorules': [ + 'div:nth-child(2n):not(.test){color:red}', + [ + [ + 'rule', + [ + [ + 'div:nth-child(2n):not(.test)', + [ + [1, 0, undefined] + ] + ] + ], + [ + [ + 'property', + [ + 'property-name', + 'color', + [ + [1, 29, undefined] + ] + ], + [ + 'property-value', + 'red', + [ + [1, 35, undefined] + ] + ] + ] + ] + ] + ] + ], + '! important': [ + 'a{color:red! important}', + [ + [ + 'rule', + [ + [ + 'a', + [ + [1, 0, undefined] + ] + ] + ], + [ + [ + 'property', + [ + 'property-name', + 'color', + [ + [1, 2, undefined] + ] + ], + [ + 'property-value', + 'red!', + [ + [1, 8, undefined] + ] + ], + [ + 'property-value', + 'important', + [ + [1, 13, undefined] + ] + ] + ] + ] + ] + ] + ], + ' !important': [ + 'a{color:red !important}', + [ + [ + 'rule', + [ + [ + 'a', + [ + [1, 0, undefined] + ] + ] + ], + [ + [ + 'property', + [ + 'property-name', + 'color', + [ + [1, 2, undefined] + ] + ], + [ + 'property-value', + 'red', + [ + [1, 8, undefined] + ] + ], + [ + 'property-value', + '!important', + [ + [1, 12, undefined] + ] + ] + ] + ] + ] + ] + ], + ' ! important': [ + 'a{color:red ! important}', + [ + [ + 'rule', + [ + [ + 'a', + [ + [1, 0, undefined] + ] + ] + ], + [ + [ + 'property', + [ + 'property-name', + 'color', + [ + [1, 2, undefined] + ] + ], + [ + 'property-value', + 'red', + [ + [1, 8, undefined] + ] + ], + [ + 'property-value', + '!', + [ + [1, 12, undefined] + ] + ], + [ + 'property-value', + 'important', + [ + [1, 14, undefined] + ] + ] + ] + ] + ] + ] + ], + '@apply': [ + 'a{@apply(--rule);color:red}', + [ + [ + 'rule', + [ + [ + 'a', + [ + [1, 0, undefined] + ] + ] + ], + [ + [ + 'at-rule', + '@apply(--rule)', + [ + [1, 2, undefined] + ] + ], + [ + 'property', + [ + 'property-name', + 'color', + [ + [1, 17, undefined] + ] + ], + [ + 'property-value', + 'red', + [ + [1, 23, undefined] + ] + ] + ] + ] + ] + ] + ], + '@apply with whitespace and no semicolon': [ + 'a{ @apply(--rule) }', + [ + [ + 'rule', + [ + [ + 'a', + [ + [1, 0, undefined] + ] + ] + ], + [ + [ + 'at-rule', + '@apply(--rule)', + [ + [1, 4, undefined] + ] + ] + ] + ] + ] + ], + '@apply within a variable': [ + ':root{--layout-horizontal:{@apply(--layout)};}', + [ + [ + 'rule', + [ + [ + ':root', + [ + [1, 0, undefined] + ] + ] + ], + [ + [ + 'property', + [ + 'property-name', + '--layout-horizontal', + [ + [1, 6, undefined] + ] + ], + [ + 'property-block', + [ + [ + 'at-rule', + '@apply(--layout)', + [ + [1, 27, undefined] + ] + ] + ] + ] + ] + ] + ] + ] + ], + '@apply within a variable before properties': [ + ':root{--layout-horizontal:{@apply(--layout);color:red;display:block};}', + [ + [ + 'rule', + [ + [ + ':root', + [ + [1, 0, undefined] + ] + ] + ], + [ + [ + 'property', + [ + 'property-name', + '--layout-horizontal', + [ + [1, 6, undefined] + ] + ], + [ + 'property-block', + [ + [ + 'at-rule', + '@apply(--layout)', + [ + [1, 27, undefined] + ] + ], + [ + 'property', + [ + 'property-name', + 'color', + [ + [1, 44, undefined] + ] + ], + [ + 'property-value', + 'red', + [ + [1, 50, undefined] + ] + ] + ], + [ + 'property', + [ + 'property-name', + 'display', + [ + [1, 54, undefined] + ] + ], + [ + 'property-value', + 'block', + [ + [1, 62, undefined] + ] + ] + ] + ] + ] + ] + ] + ] + ] + ], + 'media query': [ + '@media (min-width:980px){}', + [ + [ + 'block', + [ + [ + '@media (min-width:980px)', + [ + [1, 0, undefined] + ] + ] + ], + [] + ] + ] + ], + 'multiple media query': [ + '@media print,(min-width:980px){a{color:red}}', + [ + [ + 'block', + [ + [ + '@media print', + [ + [1, 0, undefined] + ] + ], + [ + '(min-width:980px)', + [ + [1, 13, undefined] + ] + ] + ], + [ + [ + 'rule', + [ + [ + 'a', + [ + [1, 31, undefined] + ] + ] + ], + [ + [ + 'property', + [ + 'property-name', + 'color', + [ + [1, 33, undefined] + ] + ], + [ + 'property-value', + 'red', + [ + [1, 39, undefined] + ] + ] + ] + ] + ] + ] + ] + ] + ], + 'media query surrounded by rules': [ + 'a{color:red}@media (min-width:980px){}p{color:blue}', + [ + [ + 'rule', + [ + [ + 'a', + [ + [1, 0, undefined] + ] + ] + ], + [ + [ + 'property', + [ + 'property-name', + 'color', + [ + [1, 2, undefined] + ] + ], + [ + 'property-value', + 'red', + [ + [1, 8, undefined] + ] + ] + ] + ] + ], + [ + 'block', + [ + [ + '@media (min-width:980px)', + [ + [1, 12, undefined] + ] + ] + ], + [] + ], + [ + 'rule', + [ + [ + 'p', + [ + [1, 38, undefined] + ] + ] + ], + [ + [ + 'property', + [ + 'property-name', + 'color', + [ + [1, 40, undefined] + ] + ], + [ + 'property-value', + 'blue', + [ + [1, 46, undefined] + ] + ] + ] + ] + ] + ] + ], + 'media query with rules': [ + '@media (min-width:980px){a{color:red}}', + [ + [ + 'block', + [ + [ + '@media (min-width:980px)', + [ + [1, 0, undefined] + ] + ] + ], + [ + [ + 'rule', + [ + [ + 'a', + [ + [1, 25, undefined] + ] + ] + ], + [ + [ + 'property', + [ + 'property-name', + 'color', + [ + [1, 27, undefined] + ] + ], + [ + 'property-value', + 'red', + [ + [1, 33, undefined] + ] + ] + ] + ] + ] + ] + ] + ] + ], + 'nested media query': [ + '@media only screen and (max-width:1319px){\n@media print {\na{color:#000}\n}\na{color:red}}', + [ + [ + 'block', + [ + [ + '@media only screen and (max-width:1319px)', + [ + [1, 0, undefined] + ] + ] + ], + [ + [ + 'block', + [ + [ + '@media print', + [ + [2, 0, undefined] + ] + ] + ], + [ + [ + 'rule', + [ + [ + 'a', + [ + [3, 0, undefined] + ] + ] + ], + [ + [ + 'property', + [ + 'property-name', + 'color', + [ + [3, 2, undefined] + ] + ], + [ + 'property-value', + '#000', + [ + [3, 8, undefined] + ] + ] + ] + ] + ] + ] + ], + [ + 'rule', + [ + [ + 'a', + [ + [5, 0, undefined] + ] + ] + ], + [ + [ + 'property', + [ + 'property-name', + 'color', + [ + [5, 2, undefined] + ] + ], + [ + 'property-value', + 'red', + [ + [5, 8, undefined] + ] + ] + ] + ] + ] + ] + ] + ] + ], + '@media with whitespace': [ + '@media ( min-width:980px ){ }p{color:red}', + [ + [ + 'block', + [ + [ + '@media ( min-width:980px )', + [ + [1, 0, undefined] + ] + ] + ], + [] + ], + [ + 'rule', + [ + [ + 'p', + [ + [1, 29, undefined] + ] + ] + ], + [ + [ + 'property', + [ + 'property-name', + 'color', + [ + [1, 31, undefined] + ] + ], + [ + 'property-value', + 'red', + [ + [1, 37, undefined] + ] + ] + ] + ] + ] + ] + ], + 'between blocks': [ + '@media (min-width:980px){}/*! comment */@media screen{}', + [ + [ + 'block', + [ + [ + '@media (min-width:980px)', + [ + [1, 0, undefined] + ] + ] + ], + [] + ], + [ + 'comment', + [ + '/*! comment */', + [ + [1, 26, undefined] + ] + ] + ], + [ + 'block', + [ + [ + '@media screen', + [ + [1, 40, undefined] + ] + ] + ], + [] + ] + ] + ], + 'in blocks': [ + '@media (/* comment \n */min-width:980px){a{color:red}}', + [ + [ + 'comment', + [ + '/* comment \n */', + [ + [1, 8, undefined] + ] + ] + ], + [ + 'block', + [ + [ + '@media (min-width:980px)', + [ + [1, 0, undefined] + ] + ] + ], + [ + [ + 'rule', + [ + [ + 'a', + [ + [2, 20, undefined] + ] + ] + ], + [ + [ + 'property', + [ + 'property-name', + 'color', + [ + [2, 22, undefined] + ] + ], + [ + 'property-value', + 'red', + [ + [2, 28, undefined] + ] + ] + ] + ] + ] + ] + ] + ] + ], + 'font-face': [ + '@font-face{font-family: "Helvetica Neue";font-size:12px}', + [ + [ + 'at-rule-block', + [ + [ + '@font-face', + [ + [1, 0, undefined] + ] + ] + ], + [ + [ + 'property', + [ + 'property-name', + 'font-family', + [ + [1, 11, undefined] + ] + ], + [ + 'property-value', + '"Helvetica Neue"', + [ + [1, 24, undefined] + ] + ] + ], + [ + 'property', + [ + 'property-name', + 'font-size', + [ + [1, 41, undefined] + ] + ], + [ + 'property-value', + '12px', + [ + [1, 51, undefined] + ] + ] + ] + ] + ] + ] + ], + 'charset': [ + '@charset \'utf-8\';a{color:red}', + [ + [ + 'at-rule', + '@charset \'utf-8\'', + [ + [1, 0, undefined] + ] + ], + [ + 'rule', + [ + [ + 'a', + [ + [1, 17, undefined] + ] + ] + ], + [ + [ + 'property', + [ + 'property-name', + 'color', + [ + [1, 19, undefined] + ] + ], + [ + 'property-value', + 'red', + [ + [1, 25, undefined] + ] + ] + ] + ] + ] + ] + ], + 'charset after a line break': [ + '\n@charset \n\'utf-8\';', + [ + [ + 'at-rule', + '@charset \n\'utf-8\'', + [ + [2, 0, undefined] + ] + ] + ] + ], + '@import': [ + 'a{}@import \n"test.css";\n\na{color:red}', + [ + [ + 'rule', + [ + [ + 'a', + [ + [1, 0, undefined] + ] + ] + ], + [] + ], + [ + 'at-rule', + '@import \n"test.css"', + [ + [1, 3, undefined] + ] + ], + [ + 'rule', + [ + [ + 'a', + [ + [4, 0, undefined] + ] + ] + ], + [ + [ + 'property', + [ + 'property-name', + 'color', + [ + [4, 2, undefined] + ] + ], + [ + 'property-value', + 'red', + [ + [4, 8, undefined] + ] + ] + ] + ] + ] + ] + ], + '@import with round braces': [ + '@import url(http://fonts.googleapis.com/css?family=Lora:400,700);', + [ + [ + 'at-rule', + '@import url(http://fonts.googleapis.com/css?family=Lora:400,700)', + [[1, 0, undefined]] + ] + ] + ], + '@import with media': [ + '@import "test.css" screen, tv, print;', + [ + [ + 'at-rule', + '@import "test.css" screen, tv, print', + [[1, 0, undefined]] + ] + ] + ], + 'keyframes with quoted name': [ + '@keyframes "test"{0%{color:red}}', + [ + [ + 'block', + [ + [ + '@keyframes "test"', + [ + [1, 0, undefined] + ] + ] + ], + [ + [ + 'rule', + [ + [ + '0%', + [ + [1, 18, undefined] + ] + ] + ], + [ + [ + 'property', + [ + 'property-name', + 'color', + [ + [1, 21, undefined] + ] + ], + [ + 'property-value', + 'red', + [ + [1, 27, undefined] + ] + ] + ] + ] + ] + ] + ] + ] + ], + 'variables': [ + 'a{border:var(--width)var(--style)var(--color)}', + [ + [ + 'rule', + [ + [ + 'a', + [ + [1, 0, undefined] + ] + ] + ], + [ + [ + 'property', + [ + 'property-name', + 'border', + [ + [1, 2, undefined] + ] + ], + [ + 'property-value', + 'var(--width)', + [ + [1, 9, undefined] + ] + ], + [ + 'property-value', + 'var(--style)', + [ + [1, 21, undefined] + ] + ], + [ + 'property-value', + 'var(--color)', + [ + [1, 33, undefined] + ] + ] + ] + ] + ] + ] + ], + 'variable declarations': [ + ':root{--color:var(--otherColor)}', + [ + [ + 'rule', + [ + [ + ':root', + [ + [1, 0, undefined] + ] + ] + ], + [ + [ + 'property', + [ + 'property-name', + '--color', + [ + [1, 6, undefined] + ] + ], + [ + 'property-value', + 'var(--otherColor)', + [ + [1, 14, undefined] + ] + ] + ] + ] + ] + ] + ], + 'multiple variable blocks': [ + 'div{--test1:{color:red};--test2:{color:blue};}', + [ + [ + 'rule', + [ + [ + 'div', + [ + [1, 0, undefined] + ] + ] + ], + [ + [ + 'property', + [ + 'property-name', + '--test1', + [ + [1, 4, undefined] + ] + ], + [ + 'property-block', + [ + [ + 'property', + [ + 'property-name', + 'color', + [ + [1, 13, undefined] + ] + ], + [ + 'property-value', + 'red', + [ + [1, 19, undefined] + ] + ] + ] + ] + ] + ], + [ + 'property', + [ + 'property-name', + '--test2', + [ + [1, 24, undefined] + ] + ], + [ + 'property-block', + [ + [ + 'property', + [ + 'property-name', + 'color', + [ + [1, 33, undefined] + ] + ], + [ + 'property-value', + 'blue', + [ + [1, 39, undefined] + ] + ] + ] + ] + ] + ] + ] + ] + ] + ], + 'variable block with trailing whitespace': [ + 'a{--test:{color:#f00 };color:blue}', + [ + [ + 'rule', + [ + [ + 'a', + [ + [1, 0, undefined] + ] + ] + ], + [ + [ + 'property', + [ + 'property-name', + '--test', + [ + [1, 2, undefined] + ] + ], + [ + 'property-block', + [ + [ + 'property', + [ + 'property-name', + 'color', + [ + [1, 10, undefined] + ] + ], + [ + 'property-value', + '#f00', + [ + [1, 16, undefined] + ] + ] + ] + ] + ] + ], + [ + 'property', + [ + 'property-name', + 'color', + [ + [1, 23, undefined] + ] + ], + [ + 'property-value', + 'blue', + [ + [1, 29, undefined] + ] + ] + ] + ] + ] + ] + ], + '_:-ms-lang flat block': [ + '_:-ms-lang(x),@-ms-viewport{color:red}', + [ + [ + 'at-rule-block', + [ + [ + '_:-ms-lang(x)', + [ + [1, 0, undefined] + ] + ], + [ + '@-ms-viewport', + [ + [1, 14, undefined] + ] + ] + ], + [ + [ + 'property', + [ + 'property-name', + 'color', + [ + [1, 28, undefined] + ] + ], + [ + 'property-value', + 'red', + [ + [1, 34, undefined] + ] + ] + ] + ] + ] + ] + ] + }) + ) + .addBatch( + tokenizerContext('Polymer mixins', { + 'flat value': [ + 'a{--my-toolbar-color:red}', + [ + [ + 'rule', + [ + [ + 'a', + [ + [1, 0, undefined] + ] + ] + ], + [ + [ + 'property', + [ + 'property-name', + '--my-toolbar-color', + [ + [1, 2, undefined] + ] + ], + [ + 'property-value', + 'red', + [ + [1, 21, undefined] + ] + ] + ] + ] + ] + ] + ], + 'block value': [ + 'a{--my-toolbar:{color:red;width:100%}}', + [ + [ + 'rule', + [ + [ + 'a', + [ + [1, 0, undefined] + ] + ] + ], + [ + [ + 'property', + [ + 'property-name', + '--my-toolbar', + [ + [1, 2, undefined] + ] + ], + [ + 'property-block', + [ + [ + 'property', + [ + 'property-name', + 'color', + [ + [1, 16, undefined] + ] + ], + [ + 'property-value', + 'red', + [ + [1, 22, undefined] + ] + ] + ], + [ + 'property', + [ + 'property-name', + 'width', + [ + [1, 26, undefined] + ] + ], + [ + 'property-value', + '100%', + [ + [1, 32, undefined] + ] + ] + ] + ] + ] + ] + ] + ] + ] + ], + 'mixed block value': [ + 'a{display:block;--my-toolbar:{color:red;width:100%};color:blue}', + [ + [ + 'rule', + [ + [ + 'a', + [ + [1, 0, undefined] + ] + ] + ], + [ + [ + 'property', + [ + 'property-name', + 'display', + [ + [1, 2, undefined] + ] + ], + [ + 'property-value', + 'block', + [ + [1, 10, undefined] + ] + ] + ], + [ + 'property', + [ + 'property-name', + '--my-toolbar', + [ + [1, 16, undefined] + ] + ], + [ + 'property-block', + [ + [ + 'property', + [ + 'property-name', + 'color', + [ + [1, 30, undefined] + ] + ], + [ + 'property-value', + 'red', + [ + [1, 36, undefined] + ] + ] + ], + [ + 'property', + [ + 'property-name', + 'width', + [ + [1, 40, undefined] + ] + ], + [ + 'property-value', + '100%', + [ + [1, 46, undefined] + ] + ] + ] + ] + ] + ], + [ + 'property', + [ + 'property-name', + 'color', + [ + [1, 52, undefined] + ] + ], + [ + 'property-value', + 'blue', + [ + [1, 58, undefined] + ] + ] + ] + ] + ] + ] + ] + }) + ) + .addBatch( + tokenizerContext('multiple values', { + 'comma - no spaces': [ + 'a{background:no-repeat,no-repeat}', + [ + [ + 'rule', + [ + [ + 'a', + [ + [1, 0, undefined] + ] + ] + ], + [ + [ + 'property', + [ + 'property-name', + 'background', + [ + [1, 2, undefined] + ] + ], + [ + 'property-value', + 'no-repeat', + [ + [1, 13, undefined] + ] + ], + [ + 'property-value', + ',', + [ + [1, 22, undefined] + ] + ], + [ + 'property-value', + 'no-repeat', + [ + [1, 23, undefined] + ] + ] + ] + ] + ] + ] + ], + 'comma - one space': [ + 'a{background:no-repeat, no-repeat}', + [ + [ + 'rule', + [ + [ + 'a', + [ + [1, 0, undefined] + ] + ] + ], + [ + [ + 'property', + [ + 'property-name', + 'background', + [ + [1, 2, undefined] + ] + ], + [ + 'property-value', + 'no-repeat', + [ + [1, 13, undefined] + ] + ], + [ + 'property-value', + ',', + [ + [1, 22, undefined] + ] + ], + [ + 'property-value', + 'no-repeat', + [ + [1, 24, undefined] + ] + ] + ] + ] + ] + ] + ], + 'comma - two spaces': [ + 'a{background:no-repeat , no-repeat}', + [ + [ + 'rule', + [ + [ + 'a', + [ + [1, 0, undefined] + ] + ] + ], + [ + [ + 'property', + [ + 'property-name', + 'background', + [ + [1, 2, undefined] + ] + ], + [ + 'property-value', + 'no-repeat', + [ + [1, 13, undefined] + ] + ], + [ + 'property-value', + ',', + [ + [1, 23, undefined] + ] + ], + [ + 'property-value', + 'no-repeat', + [ + [1, 25, undefined] + ] + ] + ] + ] + ] + ] + ], + 'comma - inside function': [ + 'a{background:rgba(0,0,0,0)}', + [ + [ + 'rule', + [ + [ + 'a', + [ + [1, 0, undefined] + ] + ] + ], + [ + [ + 'property', + [ + 'property-name', + 'background', + [ + [1, 2, undefined] + ] + ], + [ + 'property-value', + 'rgba(0,0,0,0)', + [ + [1, 13, undefined] + ] + ] + ] + ] + ] + ] + ], + 'forward slash - no spaces': [ + 'a{border-radius:5px/4px}', + [ + [ + 'rule', + [ + [ + 'a', + [ + [1, 0, undefined] + ] + ] + ], + [ + [ + 'property', + [ + 'property-name', + 'border-radius', + [ + [1, 2, undefined] + ] + ], + [ + 'property-value', + '5px', + [ + [1, 16, undefined] + ] + ], + [ + 'property-value', + '/', + [ + [1, 19, undefined] + ] + ], + [ + 'property-value', + '4px', + [ + [1, 20, undefined] + ] + ] + ] + ] + ] + ] + ], + 'forward slash - one space': [ + 'a{border-radius:5px /4px}', + [ + [ + 'rule', + [ + [ + 'a', + [ + [1, 0, undefined] + ] + ] + ], + [ + [ + 'property', + [ + 'property-name', + 'border-radius', + [ + [1, 2, undefined] + ] + ], + [ + 'property-value', + '5px', + [ + [1, 16, undefined] + ] + ], + [ + 'property-value', + '/', + [ + [1, 20, undefined] + ] + ], + [ + 'property-value', + '4px', + [ + [1, 21, undefined] + ] + ] + ] + ] + ] + ] + ], + 'forward slash - two spaces': [ + 'a{border-radius:5px / 4px}', + [ + [ + 'rule', + [ + [ + 'a', + [ + [1, 0, undefined] + ] + ] + ], + [ + [ + 'property', + [ + 'property-name', + 'border-radius', + [ + [1, 2, undefined] + ] + ], + [ + 'property-value', + '5px', + [ + [1, 16, undefined] + ] + ], + [ + 'property-value', + '/', + [ + [1, 20, undefined] + ] + ], + [ + 'property-value', + '4px', + [ + [1, 22, undefined] + ] + ] + ] + ] + ] + ] + ], + 'forward slash - inside function': [ + 'a{width:calc(5px/4px)}', + [ + [ + 'rule', + [ + [ + 'a', + [ + [1, 0, undefined] + ] + ] + ], + [ + [ + 'property', + [ + 'property-name', + 'width', + [ + [1, 2, undefined] + ] + ], + [ + 'property-value', + 'calc(5px/4px)', + [ + [1, 8, undefined] + ] + ] + ] + ] + ] + ] + ], + 'forward slash and closing round brace inside function': [ + 'a{width:calc((10rem - 2px) / 2 + 10em)}', + [ + [ + 'rule', + [ + [ + 'a', + [ + [1, 0, undefined] + ] + ] + ], + [ + [ + 'property', + [ + 'property-name', + 'width', + [ + [1, 2, undefined] + ] + ], + [ + 'property-value', + 'calc((10rem - 2px) / 2 + 10em)', + [ + [1, 8, undefined] + ] + ] + ] + ] + ] + ] + ], + 'quotes inside function': [ + 'a{width:expression(this.parentNode.innerText == ")" ? "5px" : "10px")}', + [ + [ + 'rule', + [ + [ + 'a', + [ + [1, 0, undefined] + ] + ] + ], + [ + [ + 'property', + [ + 'property-name', + 'width', + [ + [1, 2, undefined] + ] + ], + [ + 'property-value', + 'expression(this.parentNode.innerText == ")" ? "5px" : "10px")', + [ + [1, 8, undefined] + ] + ] + ] + ] + ] + ] + ], + 'curly braces inside function': [ + 'a{zoom:expression(function (el){el.style.zoom="1"}(this))}', + [ + [ + 'rule', + [ + [ + 'a', + [ + [1, 0, undefined] + ] + ] + ], + [ + [ + 'property', + [ + 'property-name', + 'zoom', + [ + [1, 2, undefined] + ] + ], + [ + 'property-value', + 'expression(function (el){el.style.zoom="1"}(this))', + [ + [1, 7, undefined] + ] + ] + ] + ] + ] + ] + ] + }) + ) + .addBatch( + tokenizerContext('broken', { + 'missing end brace': [ + 'a{display:block', + [ + [ + 'rule', + [ + [ + 'a', + [ + [1, 0, undefined] + ] + ] + ], + [ + [ + 'property', + [ + 'property-name', + 'display', + [ + [1, 2, undefined] + ] + ], + [ + 'property-value', + 'block', + [ + [1, 10, undefined] + ] + ] + ] + ] + ] + ] + ], + 'missing closing bracket': [ + 'a{width:expression(this.parentNode.innerText == }', + [ + [ + 'rule', + [ + [ + 'a', + [ + [1, 0, undefined] + ] + ] + ], + [ + [ + 'property', + [ + 'property-name', + 'width', + [ + [1, 2, undefined] + ] + ], + [ + 'property-value', + 'expression(this.parentNode.innerText ==', + [ + [1, 8, undefined] + ] + ] + ] + ] + ] + ] + ], + 'missing end brace in the middle': [ + 'body{color:red;a{color:blue;}', + [ + [ + 'rule', + [ + [ + 'body', + [ + [1, 0, undefined] + ] + ] + ], + [ + [ + 'property', + [ + 'property-name', + 'color', + [ + [1, 5, undefined] + ] + ], + [ + 'property-value', + 'red', + [ + [1, 11, undefined] + ] + ] + ], + [ + 'property', + [ + 'property-name', + 'a{color', + [ + [1, 15, undefined] + ] + ], + [ + 'property-value', + 'blue', + [ + [1, 23, undefined] + ] + ] + ] + ] + ] + ] + ], + 'extra end brace in the middle': [ + 'body{color:red}}a{color:blue;}', + [ + [ + 'rule', + [ + [ + 'body', + [ + [1, 0, undefined] + ] + ] + ], + [ + [ + 'property', + [ + 'property-name', + 'color', + [ + [1, 5, undefined] + ] + ], + [ + 'property-value', + 'red', + [ + [1, 11, undefined] + ] + ] + ] + ] + ], + [ + 'rule', + [ + [ + 'a', + [ + [1, 16, undefined] + ] + ] + ], + [ + [ + 'property', + [ + 'property-name', + 'color', + [ + [1, 18, undefined] + ] + ], + [ + 'property-value', + 'blue', + [ + [1, 24, undefined] + ] + ] + ] + ] + ] + ] + ] + }) + ) + .addBatch({ + 'warnings': { + 'topic': function () { + var warnings = []; + + tokenize('a{display:block', { + source: 'one.css', + options: {}, + warnings: warnings + }); + + return warnings; + }, + 'logs them correctly': function (warnings) { + assert.deepEqual(warnings, ['Missing \'}\' at line 1, column 15']); + } + } + }) + .addBatch({ + 'sources - rule with properties': { + 'topic': function () { + return tokenize('a{color:red}', { + source: 'one.css', + options: {}, + warnings: [] + }); + }, + 'sets source correctly': function (tokens) { + assert.deepEqual(tokens, [ + [ + 'rule', + [ + [ + 'a', + [ + [1, 0, 'one.css'] + ] + ] + ], + [ + [ + 'property', + [ + 'property-name', + 'color', + [ + [1, 2, 'one.css'] + ] + ], + [ + 'property-value', + 'red', + [ + [1, 8, 'one.css'] + ] + ] + ] + ] + ] + ]); + } + } + }) + .addBatch({ + 'input source maps - simple': { + 'topic': function () { + var sourceMapTracker = inputSourceMapTracker({ + errors: {} + }); + sourceMapTracker.track('styles.css', inputMap); + + return tokenize('div > a {\n color: red;\n}', { + source: 'styles.css', + inputSourceMap: true, + inputSourceMapTracker: sourceMapTracker, + options: {}, + warnings: [] + }); + }, + 'sets positions correctly': function (tokens) { + assert.deepEqual(tokens, [ + [ + 'rule', + [ + [ + 'div > a', + [ + [1, 0, 'styles.less'] + ] + ] + ], + [ + [ + 'property', + [ + 'property-name', + 'color', + [ + [2, 2, 'styles.less'] + ] + ], + [ + 'property-value', + 'red', + [ + [2, 2, 'styles.less'] + ] + ] + ] + ] + ] + ]); + } + }, + 'with fallback for properties': { + 'topic': function () { + var sourceMapTracker = inputSourceMapTracker({ + errors: {} + }); + sourceMapTracker.track('styles.css', inputMap); + + return tokenize('div > a {\n color: red red;\n}', { + source: 'styles.css', + inputSourceMap: true, + inputSourceMapTracker: sourceMapTracker, + options: {}, + warnings: [] + }); + }, + 'sets positions correctly': function (tokens) { + assert.deepEqual(tokens, [ + [ + 'rule', + [ + [ + 'div > a', + [ + [1, 0, 'styles.less'] + ] + ] + ], + [ + [ + 'property', + [ + 'property-name', + 'color', + [ + [2, 2, 'styles.less'] + ] + ], + [ + 'property-value', + 'red', + [ + [2, 2, 'styles.less'] + ] + ], + [ + 'property-value', + 'red', + [ + [2, 2, 'styles.less'] + ] + ] + ] + ] + ] + ]); + } + } + }) + .export(module); diff --git a/test/tokenizer/tokenizer-source-maps-test.js b/test/tokenizer/tokenizer-source-maps-test.js deleted file mode 100644 index 0fd62f34..00000000 --- a/test/tokenizer/tokenizer-source-maps-test.js +++ /dev/null @@ -1,582 +0,0 @@ -var vows = require('vows'); -var assert = require('assert'); -var tokenize = require('../../lib/tokenizer/tokenize'); -var SourceTracker = require('../../lib/utils/source-tracker'); -var SourceReader = require('../../lib/utils/source-reader'); -var InputSourceMapTracker = require('../../lib/utils/input-source-map-tracker'); - -var fs = require('fs'); -var path = require('path'); -var inputMapPath = path.join('test', 'fixtures', 'source-maps', 'styles.css.map'); -var inputMap = fs.readFileSync(inputMapPath, 'utf-8'); - -function sourceMapContext(group, specs) { - var ctx = {}; - - function tokenizedContext(target) { - return function (tokenized) { - assert.deepEqual(tokenized, target); - }; - } - - function toTokens(source) { - return function () { - return tokenize(source, { - sourceTracker: sourceTracker, - sourceReader: sourceReader, - inputSourceMapTracker: inputSourceMapTracker, - options: { sourceMap: true } - }); - }; - } - - for (var test in specs) { - var target = specs[test][1]; - var sourceTracker = new SourceTracker(); - var sourceReader = new SourceReader(); - var inputSourceMapTracker = new InputSourceMapTracker({ - options: { inliner: {} }, - errors: {}, - sourceTracker: sourceTracker - }); - - ctx[group + ' ' + test] = { - topic: typeof specs[test][0] == 'function' ? - specs[test][0] : - toTokens(specs[test][0]), - tokenized: tokenizedContext(target) - }; - } - - return ctx; -} - -vows.describe('source-maps/analyzer') - .addBatch( - sourceMapContext('selectors', { - 'single': [ - 'a{}', - [ - [ - 'selector', - [['a', [[1, 0, undefined]]]], - [] - ] - ] - ], - 'double': [ - 'a,div{}', - [ - [ - 'selector', - [ - ['a', [[1, 0, undefined]]], - ['div', [[1, 2, undefined]]] - ], - [] - ] - ] - ], - 'double with whitespace': [ - ' a,\n\ndiv{}', - [ - [ - 'selector', - [['a', [[1, 1, undefined]]], ['div', [[3, 0, undefined]]]], - [] - ] - ] - ], - 'triple': [ - 'a,div,p{}', - [ - [ - 'selector', - [['a', [[1, 0, undefined]]], ['div', [[1, 2, undefined]]], ['p', [[1, 6, undefined]]]], - [] - ] - ] - ], - 'triple with whitespace': [ - ' a,\n\ndiv\na,\n p{}', - [ - [ - 'selector', - [['a', [[1, 1, undefined]]], ['div\na', [[3, 0, undefined]]], ['p', [[5, 1, undefined]]]], - [] - ] - ] - ], - 'two': [ - 'a{}div{}', - [ - [ - 'selector', - [['a', [[1, 0, undefined]]]], - [] - ], - [ - 'selector', - [['div', [[1, 3, undefined]]]], - [] - ] - ] - ], - 'three with whitespace and breaks': [ - 'a {}\n\ndiv{}\n \n p{}', - [ - [ - 'selector', - [['a', [[1, 0, undefined]]]], - [] - ], - [ - 'selector', - [['div', [[3, 0, undefined]]]], - [] - ], - [ - 'selector', - [['p', [[5, 2, undefined]]]], - [] - ] - ] - ] - }) - ) - .addBatch( - sourceMapContext('properties', { - 'single': [ - 'a{color:red}', - [ - [ - 'selector', - [['a', [[1, 0, undefined]]]], - [[['color', [[1, 2, undefined]]], ['red', [[1, 8, undefined]]]]] - ] - ] - ], - 'double': [ - 'a{color:red;border:none}', - [ - [ - 'selector', - [['a', [[1, 0, undefined]]]], - [ - [['color', [[1, 2, undefined]]], ['red', [[1, 8, undefined]]]], - [['border', [[1, 12, undefined]]], ['none', [[1, 19, undefined]]]] - ] - ] - ] - ], - 'triple with whitespace': [ - 'a{color:red;\nborder:\nnone;\n\n display:block}', - [ - [ - 'selector', - [['a', [[1, 0, undefined]]]], - [ - [['color', [[1, 2, undefined]]], ['red', [[1, 8, undefined]]]], - [['border', [[2, 0, undefined]]], ['none', [[3, 0, undefined]]]], - [['display', [[5, 2, undefined]]], ['block', [[5, 10, undefined]]]] - ] - ] - ] - ], - 'two declarations': [ - 'a{color:red}div{color:blue}', - [ - [ - 'selector', - [['a', [[1, 0, undefined]]]], - [[['color', [[1, 2, undefined]]], ['red', [[1, 8, undefined]]]]] - ], - [ - 'selector', - [['div', [[1, 12, undefined]]]], - [[['color', [[1, 16, undefined]]], ['blue', [[1, 22, undefined]]]]] - ] - ] - ], - 'two declarations with whitespace': [ - 'a{color:red}\n div{color:blue}', - [ - [ - 'selector', - [['a', [[1, 0, undefined]]]], - [[['color', [[1, 2, undefined]]], ['red', [[1, 8, undefined]]]]] - ], - [ - 'selector', - [['div', [[2, 1, undefined]]]], - [[['color', [[2, 5, undefined]]], ['blue', [[2, 11, undefined]]]]] - ] - ] - ], - 'two declarations with whitespace and ending semicolon': [ - 'a{color:red;\n}\n div{color:blue}', - [ - [ - 'selector', - [['a', [[1, 0, undefined]]]], - [[['color', [[1, 2, undefined]]], ['red', [[1, 8, undefined]]]]] - ], - [ - 'selector', - [['div', [[3, 1, undefined]]]], - [[['color', [[3, 5, undefined]]], ['blue', [[3, 11, undefined]]]]] - ] - ] - ] - }) - ) - .addBatch( - sourceMapContext('at rules', { - '@import': [ - 'a{}@import \n"test.css";\n\na{color:red}', - [ - [ - 'selector', - [['a', [[1, 0, undefined]]]], - [] - ], - [ - 'at-rule', - ['@import \n"test.css";', [[1, 3, undefined]]] - ], - [ - 'selector', - [['a', [[4, 0, undefined]]]], - [[['color', [[4, 2, undefined]]], ['red', [[4, 8, undefined]]]]] - ] - ] - ], - '@charset': [ - '@charset "utf-8";a{color:red}', - [ - [ - 'at-rule', - ['@charset "utf-8";', [[1, 0, undefined]]] - ], - [ - 'selector', - [['a', [[1, 18, undefined]]]], - [[['color', [[1, 20, undefined]]], ['red', [[1, 26, undefined]]]]] - ] - ] - ] - }) - ) - .addBatch( - sourceMapContext('blocks', { - '@media - simple': [ - '@media (min-width:980px){a{color:red}}', - [ - [ - 'block', - ['@media (min-width:980px)', [[1, 0, undefined]]], - [ - [ - 'selector', - [['a', [[1, 25, undefined]]]], - [[['color', [[1, 27, undefined]]], ['red', [[1, 33, undefined]]]]] - ] - ] - ] - ] - ], - '@media - with whitespace': [ - '@media (\nmin-width:980px)\n{\na{\ncolor:\nred}p{}}', - [ - [ - 'block', - ['@media (\nmin-width:980px)', [[1, 0, undefined]]], - [ - [ - 'selector', - [['a', [[4, 0, undefined]]]], - [[['color', [[5, 0, undefined]]], ['red', [[6, 0, undefined]]]]] - ], - [ - 'selector', - [['p', [[6, 4, undefined]]]], - [] - ] - ] - ] - ] - ], - '@media - stray whitespace at end': [ - '@media (min-width:980px){a{color:red} }p{color:red}', - [ - [ - 'block', - ['@media (min-width:980px)', [[1, 0, undefined]]], - [ - [ - 'selector', - [['a', [[1, 25, undefined]]]], - [[['color', [[1, 27, undefined]]], ['red', [[1, 33, undefined]]]]] - ] - ] - ], - [ - 'selector', - [['p', [[1, 39, undefined]]]], - [[['color', [[1, 41, undefined]]], ['red', [[1, 47, undefined]]]]] - ] - ] - ], - '@font-face': [ - '@font-face{font-family: "Font";\nsrc: url("font.ttf");\nfont-weight: normal;font-style: normal}a{}', - [ - [ - 'flat-block', - ['@font-face', [[1, 0, undefined]]], - [ - [['font-family', [[1, 11, undefined]]], ['"Font"', [[1, 24, undefined]]]], - [['src', [[2, 0, undefined]]], ['url("font.ttf")', [[2, 5, undefined]]]], - [['font-weight', [[3, 0, undefined]]], ['normal', [[3, 13, undefined]]]], - [['font-style', [[3, 20, undefined]]], ['normal', [[3, 32, undefined]]]] - ] - ], - [ - 'selector', - [['a', [[3, 39, undefined]]]], - [] - ] - ] - ], - '@font-face with breaks': [ - '\n@font-face\n{font-family: "Font"}', - [ - [ - 'flat-block', - ['@font-face', [[2, 0, undefined]]], - [ - [['font-family', [[3, 1, undefined]]], ['"Font"', [[3, 14, undefined]]]] - ] - ] - ] - ], - 'variable blocks': [ - '\ndiv {\n--test1:{\n color:red}; --test2:{ color: blue };}', - [ - [ - 'selector', - [['div', [[2, 0, undefined]]]], - [ - [['--test1', [[3, 0, undefined]]], [[['color', [[4, 1, undefined]]], ['red', [[4, 7, undefined]] ]]]], - [['--test2', [[4, 13, undefined]]], [[['color', [[4, 23, undefined]]], ['blue', [[4, 30, undefined]] ]]]] - ] - ] - ] - ] - }) - ) - .addBatch( - sourceMapContext('escaped content', { - 'top-level': [ - '__ESCAPED_COMMENT_CLEAN_CSS0(0, 5)__a{}', - [ - [ - 'selector', - [['a', [[1, 5, undefined]]]], - [] - ] - ] - ], - 'top-level with line breaks': [ - '__ESCAPED_COMMENT_CLEAN_CSS0(2, 5)__a{}', - [ - [ - 'selector', - [['a', [[3, 5, undefined]]]], - [] - ] - ] - ], - 'in selectors': [ - 'div[data-type=__ESCAPED_FREE_TEXT_CLEAN_CSS0(1,3)__],div[data-id=__ESCAPED_FREE_TEXT_CLEAN_CSS1(0,7)__]{color:red}', - [ - [ - 'selector', - [ - ['div[data-type=__ESCAPED_FREE_TEXT_CLEAN_CSS0(1,3)__]', [[1, 0, undefined]]], - ['div[data-id=__ESCAPED_FREE_TEXT_CLEAN_CSS1(0,7)__]', [[2, 5, undefined]]] - ], - [[['color', [[2, 26, undefined]]], ['red', [[2, 32, undefined]]]]] - ] - ] - ], - 'in properties': [ - 'div{__ESCAPED_COMMENT_SPECIAL_CLEAN_CSS0(2,5)__background:__ESCAPED_URL_CLEAN_CSS0(0,20)__;color:blue}a{font-family:__ESCAPED_FREE_TEXT_CLEAN_CSS0(1,3)__;color:red}', - [ - [ - 'selector', - [['div', [[1, 0, undefined]]]], - [ - '__ESCAPED_COMMENT_SPECIAL_CLEAN_CSS0(2,5)__', - [['background', [[3, 5, undefined]]], ['__ESCAPED_URL_CLEAN_CSS0(0,20)__', [[3, 16, undefined]]]], - [['color', [[3, 37, undefined]]], ['blue', [[3, 43, undefined]]]] - ] - ], - [ - 'selector', - [['a', [[3, 48, undefined]]]], - [ - [['font-family', [[3, 50, undefined]]], ['__ESCAPED_FREE_TEXT_CLEAN_CSS0(1,3)__', [[3, 62, undefined]]]], - [['color', [[4, 4, undefined]]], ['red', [[4, 10, undefined]]]] - ] - ] - ] - ], - 'in properties generated for closing brace': [ - 'div{background:url(image.png)no-repeat}', - [ - [ - 'selector', - [['div', [[1, 0, undefined]]]], - [[ - ['background', [[1, 4, undefined]]], - ['url(image.png)', [[1, 15, undefined]]], - ['no-repeat', [[1, 29, undefined]]] - ]] - ] - ] - ], - 'in at-rules': [ - '@charset __ESCAPED_FREE_TEXT_CLEAN_CSS0(1, 5)__;div{}', - [ - [ - 'at-rule', - ['@charset __ESCAPED_FREE_TEXT_CLEAN_CSS0(1, 5)__;', [[1, 0, undefined]]] - ], - [ - 'selector', - [['div', [[2, 7, undefined]]]], - [] - ] - ] - ], - 'in blocks': [ - '@media (__ESCAPED_COMMENT_CLEAN_CSS0(2, 1)__min-width:980px){a{color:red}}', - [ - [ - 'block', - ['@media (__ESCAPED_COMMENT_CLEAN_CSS0(2, 1)__min-width:980px)', [[1, 0, undefined]]], - [ - [ - 'selector', - [['a', [[3, 18, undefined]]]], - [[['color', [[3, 20, undefined]]], ['red', [[3, 26, undefined]]]]] - ] - ] - ] - ] - ] - }) - ) - .addBatch( - sourceMapContext('sources', { - 'one': [ - function () { - var tracker = new SourceTracker(); - var reader = new SourceReader(); - var inputTracker = new InputSourceMapTracker({ options: { inliner: {} }, errors: {}, sourceTracker: tracker }); - - var data = tracker.store('one.css', 'a{}'); - return tokenize(data, { - sourceTracker: tracker, - sourceReader: reader, - inputSourceMapTracker: inputTracker, - options: { sourceMap: true } - }); - }, - [ - [ - 'selector', - [['a', [[1, 0, 'one.css']]]], - [] - ] - ] - ], - 'two': [ - function () { - var tracker = new SourceTracker(); - var reader = new SourceReader(); - var inputTracker = new InputSourceMapTracker({ options: { inliner: {} }, errors: {}, sourceTracker: tracker }); - - var data1 = tracker.store('one.css', 'a{}'); - var data2 = tracker.store('two.css', '\na{color:red}'); - return tokenize(data1 + data2, { - sourceTracker: tracker, - sourceReader: reader, - inputSourceMapTracker: inputTracker, - options: { sourceMap: true } - }); - }, - [ - [ - 'selector', - [['a', [[1, 0, 'one.css']]]], - [] - ], - [ - 'selector', - [['a', [[2, 0, 'two.css']]]], - [[['color', [[2, 2, 'two.css']]], ['red', [[2, 8, 'two.css']]]]] - ] - ] - ] - }) - ) - .addBatch( - sourceMapContext('input source maps', { - 'one': [ - function () { - var tracker = new SourceTracker(); - var reader = new SourceReader(); - var inputTracker = new InputSourceMapTracker({ options: { inliner: {}, sourceMap: inputMap, options: {} }, errors: {}, sourceTracker: tracker }); - inputTracker.track('', function () {}); - - return tokenize('div > a {\n color: red;\n}', { - sourceTracker: tracker, - sourceReader: reader, - inputSourceMapTracker: inputTracker, - options: { sourceMap: true } - }); - }, - [ - [ - 'selector', - [['div > a', [[1, 4, 'styles.less']]]], - [[['color', [[2, 2, 'styles.less']]], ['red', [[2, 2, 'styles.less']]]]] - ] - ] - ], - 'with fallback for properties': [ - function () { - var tracker = new SourceTracker(); - var reader = new SourceReader(); - var inputTracker = new InputSourceMapTracker({ options: { inliner: {}, sourceMap: inputMap, options: {} }, errors: {}, sourceTracker: tracker }); - inputTracker.track('', function () {}); - - return tokenize('div > a {\n color: red red;\n}', { - sourceTracker: tracker, - sourceReader: reader, - inputSourceMapTracker: inputTracker, - options: { sourceMap: true } - }); - }, - [ - [ - 'selector', - [['div > a', [[1, 4, 'styles.less']]]], - [[['color', [[2, 2, 'styles.less']]], ['red', [[2, 2, 'styles.less']]], ['red', [[2, 2, 'styles.less']]]]] - ] - ] - ] - }) - ) - .export(module); diff --git a/test/tokenizer/tokenizer-test.js b/test/tokenizer/tokenizer-test.js deleted file mode 100644 index 735e3eb1..00000000 --- a/test/tokenizer/tokenizer-test.js +++ /dev/null @@ -1,407 +0,0 @@ -var vows = require('vows'); -var assert = require('assert'); -var tokenize = require('../../lib/tokenizer/tokenize'); -var SourceTracker = require('../../lib/utils/source-tracker'); - -function tokenizerContext(name, specs) { - var ctx = {}; - - function tokenized(target) { - return function (source) { - var tokens = tokenize(source, { - options: {}, - sourceTracker: new SourceTracker(), - warnings: [] - }); - - assert.deepEqual(tokens, target); - }; - } - - for (var test in specs) { - ctx[test] = { - topic: specs[test][0], - tokenized: tokenized(specs[test][1]) - }; - } - - return ctx; -} - -vows.describe(tokenize) - .addBatch( - tokenizerContext('basic', { - 'no content': [ - '', - [] - ], - 'an escaped content': [ - '__ESCAPED_COMMENT_CLEAN_CSS0__', - [] - ], - 'an escaped content followed by a break': [ - '__ESCAPED_COMMENT_CLEAN_CSS0__\n', - [] - ], - 'an empty selector': [ - 'a{}', - [ - ['selector', [['a']], []] - ] - ], - 'an empty selector with whitespace': [ - 'a{ \n }', - [ - ['selector', [['a']], []] - ] - ], - 'a selector with a property': [ - 'a{color:red}', - [ - ['selector', [['a']], [[['color'], ['red']]]] - ] - ], - 'a selector with a multi value property': [ - 'a{margin:0px 2px 1px}', - [ - ['selector', [['a']], [[['margin'], ['0px'], ['2px'], ['1px']]]] - ] - ], - 'a selector with whitespace': [ - 'a {color:red;\n\ndisplay :\r\n block }', - [ - ['selector', [['a']], [[['color'], ['red']], [['display'], ['block']]]] - ] - ], - 'a selector with suffix whitespace': [ - 'div a{color:red\r\n}', - [ - ['selector', [['div a']], [[['color'], ['red']]]] - ] - ], - 'a selector with whitespace in functions': [ - 'a{color:rgba( 255, 255, 0, 0.5 )}', - [ - ['selector', [['a']], [[['color'], ['rgba( 255, 255, 0, 0.5 )']]]] - ] - ], - 'a selector with functions and no whitespace breaks': [ - 'a{background:rgba(255,255,0,0.5)url(test.png)repeat no-repeat}', - [ - ['selector', [['a']], [[['background'], ['rgba(255,255,0,0.5)'], ['url(test.png)'], ['repeat'], ['no-repeat']]]] - ] - ], - 'a selector with escaped url and no whitespace breaks': [ - 'a{background:__ESCAPED_URL_CLEAN_CSS0__50px/25%}', - [ - ['selector', [['a']], [[['background'], ['__ESCAPED_URL_CLEAN_CSS0__'], ['50px'], ['/'], ['25%']]]] - ] - ], - 'a selector with empty properties': [ - 'a{color:red; ; ; ;}', - [ - ['selector', [['a']], [[['color'], ['red']]]] - ] - ], - 'a selector with quoted attribute': [ - 'a[data-kind=__ESCAPED_FREE_TEXT_CLEAN_CSS0__]{color:red}', - [ - ['selector', [['a[data-kind=__ESCAPED_FREE_TEXT_CLEAN_CSS0__]']], [[['color'], ['red']]]] - ] - ], - 'a selector with escaped quote': [ - '.this-class\\\'s-got-an-apostrophe{}', - [ - ['selector', [['.this-class\\\'s-got-an-apostrophe']], []] - ] - ], - 'a double selector': [ - 'a,\n\ndiv.class > p {color:red}', - [ - ['selector', [['a'], ['div.class > p']], [[['color'], ['red']]]] - ] - ], - 'two selectors': [ - 'a{color:red}div{color:blue}', - [ - ['selector', [['a']], [[['color'], ['red']]]], - ['selector', [['div']], [[['color'], ['blue']]]], - ] - ], - 'two comments and a selector separated by newline': [ - '__ESCAPED_COMMENT_CLEAN_CSS0__\n__ESCAPED_COMMENT_CLEAN_CSS1__\ndiv{}', - [ - ['selector', [['div']], []] - ] - ], - 'two properties wrapped between comments': [ - 'div{__ESCAPED_COMMENT_SPECIAL_CLEAN_CSS0__color:red__ESCAPED_COMMENT_SPECIAL_CLEAN_CSS1__}', - [ - ['selector', [['div']], ['__ESCAPED_COMMENT_SPECIAL_CLEAN_CSS0__', '__ESCAPED_COMMENT_SPECIAL_CLEAN_CSS1__', [['color'], ['red']]]] - ] - ], - 'multiple values wrapped between comments #1': [ - 'div{background:__ESCAPED_URL_CLEAN_CSS0__,__ESCAPED_COMMENT_CLEAN_CSS0__red}', - [ - ['selector', [['div']], ['__ESCAPED_COMMENT_CLEAN_CSS0__', [['background'], ['__ESCAPED_URL_CLEAN_CSS0__'], [','], ['red']]]] - ] - ], - 'multiple values wrapped between comments #2': [ - 'div{background:__ESCAPED_URL_CLEAN_CSS0__,red__ESCAPED_COMMENT_CLEAN_CSS0__}', - [ - ['selector', [['div']], ['__ESCAPED_COMMENT_CLEAN_CSS0__', [['background'], ['__ESCAPED_URL_CLEAN_CSS0__'], [','], ['red']]]] - ] - ], - 'multiple values wrapped between comments #3': [ - 'div{background:__ESCAPED_URL_CLEAN_CSS0__,rgba(0,0,0,__ESCAPED_COMMENT_CLEAN_CSS0__0.1)}', - [ - ['selector', [['div']], ['__ESCAPED_COMMENT_CLEAN_CSS0__', [['background'], ['__ESCAPED_URL_CLEAN_CSS0__'], [','], ['rgba(0,0,0,0.1)']]]] - ] - ], - 'multiple values wrapped between comments #4': [ - 'div{background:__ESCAPED_URL_CLEAN_CSS0__,rgba(0,0,0,__ESCAPED_COMMENT_CLEAN_CSS0__0.1)}', - [ - ['selector', [['div']], ['__ESCAPED_COMMENT_CLEAN_CSS0__', [['background'], ['__ESCAPED_URL_CLEAN_CSS0__'], [','], ['rgba(0,0,0,0.1)']]]] - ] - ], - 'pseudoselector after an argument one': [ - 'div:nth-child(2n):not(.test){}', - [ - ['selector', [['div:nth-child(2n):not(.test)']], []] - ] - ], - '@apply': [ - 'a{@apply(--rule);}', - [ - [ - 'selector', - [['a']], - [['at-rule', '@apply(--rule)']] - ] - ] - ], - '@apply with whitespace': [ - 'a{ @apply(--rule); }', - [ - [ - 'selector', - [['a']], - [['at-rule', '@apply(--rule)']] - ] - ] - ], - 'media query': [ - '@media (min-width:980px){}', - [ - ['block', ['@media (min-width:980px)'], []] - ] - ], - 'media query with selectors': [ - '@media (min-width:980px){a{color:red}}', - [ - ['block', ['@media (min-width:980px)'], [ - ['selector', [['a']], [[['color'], ['red']]]] - ]] - ] - ], - 'media query spanning more than one chunk': [ - '@media only screen and (max-width:1319px) and (min--moz-device-pixel-ratio:1.5),only screen and (max-width:1319px) and (-moz-min-device-pixel-ratio:1.5){a{color:#000}}', - [ - ['block', ['@media only screen and (max-width:1319px) and (min--moz-device-pixel-ratio:1.5),only screen and (max-width:1319px) and (-moz-min-device-pixel-ratio:1.5)'], [ - ['selector', [['a']], [[['color'], ['#000']]]] - ]] - ] - ], - 'font-face': [ - '@font-face{font-family: fontName;font-size:12px}', - [ - ['flat-block', ['@font-face'], [[['font-family'], ['fontName']], [['font-size'], ['12px']]]] - ] - ], - 'charset': [ - '@charset \'utf-8\';a{color:red}', - [ - ['at-rule', ['@charset \'utf-8\';']], - ['selector', [['a']], [[['color'], ['red']]]] - ] - ], - 'charset after a line break': [ - '\n@charset \n\'utf-8\';', - [ - ['at-rule', ['@charset \n\'utf-8\';']] - ] - ], - 'keyframes with quoted attribute': [ - '@keyframes __ESCAPED_FREE_TEXT_CLEAN_CSS0__{}', - [ - ['block', ['@keyframes __ESCAPED_FREE_TEXT_CLEAN_CSS0__'], []] - ] - ], - 'variables': [ - 'a{border:var(--width)var(--style)var(--color)}', - [ - [ - 'selector', - [['a']], - [[['border'], ['var(--width)'], ['var(--style)'], ['var(--color)']]] - ] - ] - ], - 'variable declarations': [ - ':root{--color:var(--otherColor)}', - [ - [ - 'selector', - [[':root']], - [[['--color'], ['var(--otherColor)']]] - ] - ] - ], - '! important': [ - 'a{color:red! important}', - [ - [ - 'selector', - [['a']], - [[['color'], ['red!important']]] - ] - ] - ], - ' !important': [ - 'a{color:red !important}', - [ - [ - 'selector', - [['a']], - [[['color'], ['red!important']]] - ] - ] - ], - ' ! important': [ - 'a{color:red ! important}', - [ - [ - 'selector', - [['a']], - [[['color'], ['red!important']]] - ] - ] - ], - '_:-ms-lang flat block': [ - '_:-ms-lang(x),@-ms-viewport{color:red}', - [ - [ - 'flat-block', - ['_:-ms-lang(x),@-ms-viewport'], - [[['color'], ['red']]] - ] - ] - ] - }) - ) - .addBatch( - tokenizerContext('Polymer mixins', { - 'flat value': [ - 'a{--my-toolbar-color:red}', - [ - ['selector', [['a']], [[['--my-toolbar-color'], ['red']]]] - ] - ], - 'block value': [ - 'a{--my-toolbar:{color:red;width:100%}}', - [ - [ - 'selector', - [['a']], - [[['--my-toolbar'], [ - [['color'], ['red']], - [['width'], ['100%']] - ]]] - ] - ] - ], - 'mixed block value': [ - 'a{display:block;--my-toolbar:{color:red;width:100%};color:blue}', - [ - [ - 'selector', - [['a']], - [ - [['display'], ['block']], - [['--my-toolbar'], [[['color'], ['red']], [['width'], ['100%']]]], - [['color'], ['blue']] - ] - ] - ] - ] - }) - ) - .addBatch( - tokenizerContext('multiple values', { - 'comma - no spaces': [ - 'a{background:no-repeat,no-repeat}', - [ - ['selector', [['a']], [[['background'], ['no-repeat'], [','], ['no-repeat']]]] - ] - ], - 'comma - one spaces': [ - 'a{background:no-repeat, no-repeat}', - [ - ['selector', [['a']], [[['background'], ['no-repeat'], [','], ['no-repeat']]]] - ] - ], - 'comma - two spaces': [ - 'a{background:no-repeat , no-repeat}', - [ - ['selector', [['a']], [[['background'], ['no-repeat'], [','], ['no-repeat']]]] - ] - ], - 'comma - inside function': [ - 'a{background:rgba(0,0,0,0)}', - [ - ['selector', [['a']], [[['background'], ['rgba(0,0,0,0)']]]] - ] - ], - 'forward slash - no spaces': [ - 'a{border-radius:5px/4px}', - [ - ['selector', [['a']], [[['border-radius'], ['5px'], ['/'], ['4px']]]] - ] - ], - 'forward slash - one spaces': [ - 'a{border-radius:5px /4px}', - [ - ['selector', [['a']], [[['border-radius'], ['5px'], ['/'], ['4px']]]] - ] - ], - 'forward slash - two spaces': [ - 'a{border-radius:5px / 4px}', - [ - ['selector', [['a']], [[['border-radius'], ['5px'], ['/'], ['4px']]]] - ] - ], - 'forward slash - inside function': [ - 'a{width:calc(5px/4px)}', - [ - ['selector', [['a']], [[['width'], ['calc(5px/4px)']]]] - ] - ], - }) - ) - .addBatch( - tokenizerContext('broken', { - 'missing end brace': [ - 'a{display:block', - [ - ['selector', [['a']], []] - ] - ], - 'missing end brace in the middle': [ - 'body{color:red;a{color:blue;}', - [ - ['selector', [['body']], [[['color'], ['red']]]] - ] - ] - }) - ) - .export(module); -- 2.34.1