From e8c2f703aec77c0f57896829adc6f79a49051e31 Mon Sep 17 00:00:00 2001 From: Jakub Pawlowicz Date: Tue, 21 Apr 2015 21:36:01 +0100 Subject: [PATCH] Moves tokenizer code into lib/tokenizer. --- History.md | 2 + lib/selectors/optimizer.js | 2 +- .../source-maps.js => source-maps/track.js} | 0 lib/{utils => tokenizer}/chunker.js | 0 lib/tokenizer/extract-properties.js | 105 +++++++++++++++ lib/tokenizer/extract-selectors.js | 17 +++ .../tokenizer.js => tokenizer/tokenize.js} | 13 +- lib/utils/extractors.js | 121 ------------------ test/properties/longhand-overriding-test.js | 2 +- test/properties/optimizer-test.js | 2 +- test/properties/override-compacting-test.js | 2 +- .../shorthand-compacting-source-maps-test.js | 2 +- test/properties/shorthand-compacting-test.js | 2 +- test/selectors/extractor-test.js | 2 +- test/selectors/optimizers/simple-test.js | 2 +- test/selectors/reorderable-test.js | 2 +- test/{utils => tokenizer}/chunker-test.js | 2 +- .../tokenizer-source-maps-test.js | 2 +- .../tokenizer-test.js | 2 +- 19 files changed, 143 insertions(+), 139 deletions(-) rename lib/{utils/source-maps.js => source-maps/track.js} (100%) rename lib/{utils => tokenizer}/chunker.js (100%) create mode 100644 lib/tokenizer/extract-properties.js create mode 100644 lib/tokenizer/extract-selectors.js rename lib/{selectors/tokenizer.js => tokenizer/tokenize.js} (94%) delete mode 100644 lib/utils/extractors.js rename test/{utils => tokenizer}/chunker-test.js (95%) rename test/{selectors => tokenizer}/tokenizer-source-maps-test.js (99%) rename test/{selectors => tokenizer}/tokenizer-test.js (99%) diff --git a/History.md b/History.md index ab15fbd2..da124b95 100644 --- a/History.md +++ b/History.md @@ -3,6 +3,8 @@ * Cleans up url rebase code getting rid of unnecessary state. * Cleans up tokenizer code getting rid of unnecessary state. +* Moves source maps tracker into lib/source-maps/track. +* Moves tokenizer code into lib/tokenizer. * Moves URL rebasing & rewriting into lib/urls. * Fixed issue [#436](https://github.com/jakubpawlowicz/clean-css/issues/436) - refactors URI rewriting. diff --git a/lib/selectors/optimizer.js b/lib/selectors/optimizer.js index e17d9a59..c804230a 100644 --- a/lib/selectors/optimizer.js +++ b/lib/selectors/optimizer.js @@ -1,4 +1,4 @@ -var tokenize = require('./tokenizer'); +var tokenize = require('../tokenizer/tokenize'); var SimpleOptimizer = require('./optimizers/simple'); var AdvancedOptimizer = require('./optimizers/advanced'); var addOptimizationMetadata = require('./optimization-metadata'); diff --git a/lib/utils/source-maps.js b/lib/source-maps/track.js similarity index 100% rename from lib/utils/source-maps.js rename to lib/source-maps/track.js diff --git a/lib/utils/chunker.js b/lib/tokenizer/chunker.js similarity index 100% rename from lib/utils/chunker.js rename to lib/tokenizer/chunker.js diff --git a/lib/tokenizer/extract-properties.js b/lib/tokenizer/extract-properties.js new file mode 100644 index 00000000..10ea3d80 --- /dev/null +++ b/lib/tokenizer/extract-properties.js @@ -0,0 +1,105 @@ +var Splitter = require('../utils/splitter'); + +var COMMA = ','; +var FORWARD_SLASH = '/'; + +function selectorName(value) { + return value[0]; +} + +function extractProperties(string, selectors, context) { + var list = []; + var splitter = new Splitter(/[ ,\/]/); + + if (typeof string != 'string') + return []; + + if (string.indexOf('__ESCAPED_COMMENT_') > -1) + string = string.replace(/(__ESCAPED_COMMENT_(SPECIAL_)?CLEAN_CSS[^_]+?__)/g, ';$1;'); + + if (string.indexOf(')') > -1) + string = string.replace(/\)([^\s_;:,\)])/g, context.sourceMap ? ') __ESCAPED_COMMENT_CLEAN_CSS(0,-1)__ $1' : ') $1'); + + if (string.indexOf('ESCAPED_URL_CLEAN_CSS') > -1) + string = string.replace(/(ESCAPED_URL_CLEAN_CSS[^_]+?__)/g, context.sourceMap ? '$1 __ESCAPED_COMMENT_CLEAN_CSS(0,-1)__ ' : '$1 '); + + var candidates = string.split(';'); + + for (var i = 0, l = candidates.length; i < l; i++) { + var candidate = candidates[i]; + var firstColonAt = candidate.indexOf(':'); + + if (firstColonAt == -1) { + context.track(candidate); + if (candidate.indexOf('__ESCAPED_COMMENT_SPECIAL') > -1) + list.push(candidate); + continue; + } + + if (candidate.indexOf('{') > 0) { + context.track(candidate); + continue; + } + + var body = []; + var name = candidate.substring(0, firstColonAt); + body.push([name.trim()].concat(context.track(name, true))); + context.track(':'); + + var values = splitter.split(candidate.substring(firstColonAt + 1), true); + + if (values.length == 1 && values[0] === '') { + context.warnings.push('Empty property \'' + name + '\' inside \'' + selectors.filter(selectorName).join(',') + '\' selector. Ignoring.'); + continue; + } + + for (var j = 0, m = values.length; j < m; j++) { + var value = values[j]; + var trimmed = value.trim(); + + if (trimmed.length === 0) + continue; + + var lastCharacter = trimmed[trimmed.length - 1]; + var endsWithNonSpaceSeparator = trimmed.length > 1 && (lastCharacter == COMMA || lastCharacter == FORWARD_SLASH); + + if (endsWithNonSpaceSeparator) + trimmed = trimmed.substring(0, trimmed.length - 1); + + if (trimmed.indexOf('__ESCAPED_COMMENT_CLEAN_CSS(0,-') > -1) { + context.track(trimmed); + continue; + } + + var pos = body.length - 1; + if (trimmed == 'important' && body[pos][0] == '!') { + context.track(trimmed); + body[pos - 1][0] += '!important'; + body.pop(); + continue; + } + + if (trimmed == '!important' || (trimmed == 'important' && body[pos][0][body[pos][0].length - 1] == '!')) { + context.track(trimmed); + body[pos][0] += trimmed; + continue; + } + + body.push([trimmed].concat(context.track(value, true))); + + if (endsWithNonSpaceSeparator) { + body.push([lastCharacter]); + context.track(lastCharacter); + } + } + + if (i < l - 1) + context.track(';'); + + list.push(body); + } + + return list; +} + +module.exports = extractProperties; diff --git a/lib/tokenizer/extract-selectors.js b/lib/tokenizer/extract-selectors.js new file mode 100644 index 00000000..cbdf367f --- /dev/null +++ b/lib/tokenizer/extract-selectors.js @@ -0,0 +1,17 @@ +var Splitter = require('../utils/splitter'); + +function extractSelectors(string, context) { + var list = []; + var metadata; + var selectors = new Splitter(',').split(string); + + for (var i = 0, l = selectors.length; i < l; i++) { + metadata = context.track(selectors[i], true, i); + context.track(','); + list.push([selectors[i].trim()].concat(metadata)); + } + + return list; +} + +module.exports = extractSelectors; diff --git a/lib/selectors/tokenizer.js b/lib/tokenizer/tokenize.js similarity index 94% rename from lib/selectors/tokenizer.js rename to lib/tokenizer/tokenize.js index ef8a2137..b779a9ce 100644 --- a/lib/selectors/tokenizer.js +++ b/lib/tokenizer/tokenize.js @@ -1,6 +1,7 @@ -var Chunker = require('../utils/chunker'); -var Extract = require('../utils/extractors'); -var track = require('../utils/source-maps'); +var Chunker = require('./chunker'); +var extractProperties = require('./extract-properties'); +var extractSelectors = require('./extract-selectors'); +var track = require('../source-maps/track'); var path = require('path'); @@ -175,7 +176,7 @@ function intoTokens(context) { newToken.push(intoTokens(context)); if (typeof newToken[2] == 'string') - newToken[2] = Extract.properties(newToken[2], [[trimmedValue]], context); + newToken[2] = extractProperties(newToken[2], [[trimmedValue]], context); context.mode = oldMode; context.track('}'); @@ -213,13 +214,13 @@ function intoTokens(context) { context.cursor = nextEnd + 2; } else if (what == 'bodyStart') { - var selectors = Extract.selectors(chunk.substring(context.cursor, nextSpecial), context); + var selectors = extractSelectors(chunk.substring(context.cursor, nextSpecial), context); oldMode = context.mode; context.cursor = nextSpecial + 1; context.mode = 'body'; - var body = Extract.properties(intoTokens(context), selectors, context); + var body = extractProperties(intoTokens(context), selectors, context); context.track('{'); context.mode = oldMode; diff --git a/lib/utils/extractors.js b/lib/utils/extractors.js deleted file mode 100644 index e4b6cdc6..00000000 --- a/lib/utils/extractors.js +++ /dev/null @@ -1,121 +0,0 @@ -var Splitter = require('./splitter'); - -var COMMA = ','; -var FORWARD_SLASH = '/'; - -function selectorName(value) { - return value[0]; -} - -var Extractors = { - properties: function (string, selectors, context) { - var list = []; - var splitter = new Splitter(/[ ,\/]/); - - if (typeof string != 'string') - return []; - - if (string.indexOf('__ESCAPED_COMMENT_') > -1) - string = string.replace(/(__ESCAPED_COMMENT_(SPECIAL_)?CLEAN_CSS[^_]+?__)/g, ';$1;'); - - if (string.indexOf(')') > -1) - string = string.replace(/\)([^\s_;:,\)])/g, context.sourceMap ? ') __ESCAPED_COMMENT_CLEAN_CSS(0,-1)__ $1' : ') $1'); - - if (string.indexOf('ESCAPED_URL_CLEAN_CSS') > -1) - string = string.replace(/(ESCAPED_URL_CLEAN_CSS[^_]+?__)/g, context.sourceMap ? '$1 __ESCAPED_COMMENT_CLEAN_CSS(0,-1)__ ' : '$1 '); - - var candidates = string.split(';'); - - for (var i = 0, l = candidates.length; i < l; i++) { - var candidate = candidates[i]; - var firstColonAt = candidate.indexOf(':'); - - if (firstColonAt == -1) { - context.track(candidate); - if (candidate.indexOf('__ESCAPED_COMMENT_SPECIAL') > -1) - list.push(candidate); - continue; - } - - if (candidate.indexOf('{') > 0) { - context.track(candidate); - continue; - } - - var body = []; - var name = candidate.substring(0, firstColonAt); - body.push([name.trim()].concat(context.track(name, true))); - context.track(':'); - - var values = splitter.split(candidate.substring(firstColonAt + 1), true); - - if (values.length == 1 && values[0] === '') { - context.warnings.push('Empty property \'' + name + '\' inside \'' + selectors.filter(selectorName).join(',') + '\' selector. Ignoring.'); - continue; - } - - for (var j = 0, m = values.length; j < m; j++) { - var value = values[j]; - var trimmed = value.trim(); - - if (trimmed.length === 0) - continue; - - var lastCharacter = trimmed[trimmed.length - 1]; - var endsWithNonSpaceSeparator = trimmed.length > 1 && (lastCharacter == COMMA || lastCharacter == FORWARD_SLASH); - - if (endsWithNonSpaceSeparator) - trimmed = trimmed.substring(0, trimmed.length - 1); - - if (trimmed.indexOf('__ESCAPED_COMMENT_CLEAN_CSS(0,-') > -1) { - context.track(trimmed); - continue; - } - - var pos = body.length - 1; - if (trimmed == 'important' && body[pos][0] == '!') { - context.track(trimmed); - body[pos - 1][0] += '!important'; - body.pop(); - continue; - } - - if (trimmed == '!important' || (trimmed == 'important' && body[pos][0][body[pos][0].length - 1] == '!')) { - context.track(trimmed); - body[pos][0] += trimmed; - continue; - } - - body.push([trimmed].concat(context.track(value, true))); - - if (endsWithNonSpaceSeparator) { - body.push([lastCharacter]); - context.track(lastCharacter); - } - } - - if (i < l - 1) - context.track(';'); - - list.push(body); - } - - return list; - }, - - selectors: function (string, context) { - var list = []; - var metadata; - var selectors = new Splitter(',').split(string); - - for (var i = 0, l = selectors.length; i < l; i++) { - metadata = context.track(selectors[i], true, i); - context.track(','); - list.push([selectors[i].trim()].concat(metadata)); - } - - return list; - } -}; - -module.exports = Extractors; diff --git a/test/properties/longhand-overriding-test.js b/test/properties/longhand-overriding-test.js index cabc8c3a..fc3d01fc 100644 --- a/test/properties/longhand-overriding-test.js +++ b/test/properties/longhand-overriding-test.js @@ -3,7 +3,7 @@ var assert = require('assert'); var optimize = require('../../lib/properties/optimizer'); -var tokenize = require('../../lib/selectors/tokenizer'); +var tokenize = require('../../lib/tokenizer/tokenize'); var SourceTracker = require('../../lib/utils/source-tracker'); var Compatibility = require('../../lib/utils/compatibility'); var Validator = require('../../lib/properties/validator'); diff --git a/test/properties/optimizer-test.js b/test/properties/optimizer-test.js index cb66aa29..e09c7feb 100644 --- a/test/properties/optimizer-test.js +++ b/test/properties/optimizer-test.js @@ -3,7 +3,7 @@ var assert = require('assert'); var optimize = require('../../lib/properties/optimizer'); -var tokenize = require('../../lib/selectors/tokenizer'); +var tokenize = require('../../lib/tokenizer/tokenize'); var SourceTracker = require('../../lib/utils/source-tracker'); var Compatibility = require('../../lib/utils/compatibility'); var Validator = require('../../lib/properties/validator'); diff --git a/test/properties/override-compacting-test.js b/test/properties/override-compacting-test.js index bdbbef32..6266a295 100644 --- a/test/properties/override-compacting-test.js +++ b/test/properties/override-compacting-test.js @@ -3,7 +3,7 @@ var assert = require('assert'); var optimize = require('../../lib/properties/optimizer'); -var tokenize = require('../../lib/selectors/tokenizer'); +var tokenize = require('../../lib/tokenizer/tokenize'); var SourceTracker = require('../../lib/utils/source-tracker'); var Compatibility = require('../../lib/utils/compatibility'); var Validator = require('../../lib/properties/validator'); diff --git a/test/properties/shorthand-compacting-source-maps-test.js b/test/properties/shorthand-compacting-source-maps-test.js index 28a6a984..341ec674 100644 --- a/test/properties/shorthand-compacting-source-maps-test.js +++ b/test/properties/shorthand-compacting-source-maps-test.js @@ -3,7 +3,7 @@ var assert = require('assert'); var optimize = require('../../lib/properties/optimizer'); -var tokenize = require('../../lib/selectors/tokenizer'); +var tokenize = require('../../lib/tokenizer/tokenize'); var SourceTracker = require('../../lib/utils/source-tracker'); var SourceReader = require('../../lib/utils/source-reader'); var InputSourceMapTracker = require('../../lib/utils/input-source-map-tracker'); diff --git a/test/properties/shorthand-compacting-test.js b/test/properties/shorthand-compacting-test.js index 2225caa9..52fa4213 100644 --- a/test/properties/shorthand-compacting-test.js +++ b/test/properties/shorthand-compacting-test.js @@ -3,7 +3,7 @@ var assert = require('assert'); var optimize = require('../../lib/properties/optimizer'); -var tokenize = require('../../lib/selectors/tokenizer'); +var tokenize = require('../../lib/tokenizer/tokenize'); var SourceTracker = require('../../lib/utils/source-tracker'); var Compatibility = require('../../lib/utils/compatibility'); var Validator = require('../../lib/properties/validator'); diff --git a/test/selectors/extractor-test.js b/test/selectors/extractor-test.js index 117b2621..ab4eb39d 100644 --- a/test/selectors/extractor-test.js +++ b/test/selectors/extractor-test.js @@ -1,6 +1,6 @@ var vows = require('vows'); var assert = require('assert'); -var tokenize = require('../../lib/selectors/tokenizer'); +var tokenize = require('../../lib/tokenizer/tokenize'); var extractor = require('../../lib/selectors/extractor'); function buildToken(source) { diff --git a/test/selectors/optimizers/simple-test.js b/test/selectors/optimizers/simple-test.js index 5eb062ca..302c3314 100644 --- a/test/selectors/optimizers/simple-test.js +++ b/test/selectors/optimizers/simple-test.js @@ -1,7 +1,7 @@ var vows = require('vows'); var assert = require('assert'); -var tokenize = require('../../../lib/selectors/tokenizer'); +var tokenize = require('../../../lib/tokenizer/tokenize'); var SimpleOptimizer = require('../../../lib/selectors/optimizers/simple'); var Compatibility = require('../../../lib/utils/compatibility'); var addOptimizationMetadata = require('../../../lib/selectors/optimization-metadata'); diff --git a/test/selectors/reorderable-test.js b/test/selectors/reorderable-test.js index 1ecdb7cd..389a33d4 100644 --- a/test/selectors/reorderable-test.js +++ b/test/selectors/reorderable-test.js @@ -1,7 +1,7 @@ var vows = require('vows'); var assert = require('assert'); -var tokenize = require('../../lib/selectors/tokenizer'); +var tokenize = require('../../lib/tokenizer/tokenize'); var extractProperties = require('../../lib/selectors/extractor'); var canReorder = require('../../lib/selectors/reorderable').canReorder; var canReorderSingle = require('../../lib/selectors/reorderable').canReorderSingle; diff --git a/test/utils/chunker-test.js b/test/tokenizer/chunker-test.js similarity index 95% rename from test/utils/chunker-test.js rename to test/tokenizer/chunker-test.js index 38966bbb..3c4861dc 100644 --- a/test/utils/chunker-test.js +++ b/test/tokenizer/chunker-test.js @@ -1,6 +1,6 @@ var vows = require('vows'); var assert = require('assert'); -var Chunker = require('../../lib/utils/chunker'); +var Chunker = require('../../lib/tokenizer/chunker'); vows.describe(Chunker) .addBatch({ diff --git a/test/selectors/tokenizer-source-maps-test.js b/test/tokenizer/tokenizer-source-maps-test.js similarity index 99% rename from test/selectors/tokenizer-source-maps-test.js rename to test/tokenizer/tokenizer-source-maps-test.js index 96f66818..97c23048 100644 --- a/test/selectors/tokenizer-source-maps-test.js +++ b/test/tokenizer/tokenizer-source-maps-test.js @@ -1,6 +1,6 @@ var vows = require('vows'); var assert = require('assert'); -var tokenize = require('../../lib/selectors/tokenizer'); +var tokenize = require('../../lib/tokenizer/tokenize'); var SourceTracker = require('../../lib/utils/source-tracker'); var SourceReader = require('../../lib/utils/source-reader'); var InputSourceMapTracker = require('../../lib/utils/input-source-map-tracker'); diff --git a/test/selectors/tokenizer-test.js b/test/tokenizer/tokenizer-test.js similarity index 99% rename from test/selectors/tokenizer-test.js rename to test/tokenizer/tokenizer-test.js index decbe91a..7a69f55b 100644 --- a/test/selectors/tokenizer-test.js +++ b/test/tokenizer/tokenizer-test.js @@ -1,6 +1,6 @@ var vows = require('vows'); var assert = require('assert'); -var tokenize = require('../../lib/selectors/tokenizer'); +var tokenize = require('../../lib/tokenizer/tokenize'); var SourceTracker = require('../../lib/utils/source-tracker'); function tokenizerContext(name, specs) { -- 2.34.1