You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

270 lines
8.8 KiB

4 years ago
  1. 'use strict';
  2. exports.__esModule = true;
  3. exports.FIELDS = undefined;
  4. var _unescapable, _wordDelimiters;
  5. exports.default = tokenize;
  6. var _tokenTypes = require('./tokenTypes');
  7. var t = _interopRequireWildcard(_tokenTypes);
  8. function _interopRequireWildcard(obj) { if (obj && obj.__esModule) { return obj; } else { var newObj = {}; if (obj != null) { for (var key in obj) { if (Object.prototype.hasOwnProperty.call(obj, key)) newObj[key] = obj[key]; } } newObj.default = obj; return newObj; } }
  9. var unescapable = (_unescapable = {}, _unescapable[t.tab] = true, _unescapable[t.newline] = true, _unescapable[t.cr] = true, _unescapable[t.feed] = true, _unescapable);
  10. var wordDelimiters = (_wordDelimiters = {}, _wordDelimiters[t.space] = true, _wordDelimiters[t.tab] = true, _wordDelimiters[t.newline] = true, _wordDelimiters[t.cr] = true, _wordDelimiters[t.feed] = true, _wordDelimiters[t.ampersand] = true, _wordDelimiters[t.asterisk] = true, _wordDelimiters[t.bang] = true, _wordDelimiters[t.comma] = true, _wordDelimiters[t.colon] = true, _wordDelimiters[t.semicolon] = true, _wordDelimiters[t.openParenthesis] = true, _wordDelimiters[t.closeParenthesis] = true, _wordDelimiters[t.openSquare] = true, _wordDelimiters[t.closeSquare] = true, _wordDelimiters[t.singleQuote] = true, _wordDelimiters[t.doubleQuote] = true, _wordDelimiters[t.plus] = true, _wordDelimiters[t.pipe] = true, _wordDelimiters[t.tilde] = true, _wordDelimiters[t.greaterThan] = true, _wordDelimiters[t.equals] = true, _wordDelimiters[t.dollar] = true, _wordDelimiters[t.caret] = true, _wordDelimiters[t.slash] = true, _wordDelimiters);
  11. var hex = {};
  12. var hexChars = "0123456789abcdefABCDEF";
  13. for (var i = 0; i < hexChars.length; i++) {
  14. hex[hexChars.charCodeAt(i)] = true;
  15. }
  16. /**
  17. * Returns the last index of the bar css word
  18. * @param {string} css The string in which the word begins
  19. * @param {number} start The index into the string where word's first letter occurs
  20. */
  21. function consumeWord(css, start) {
  22. var next = start;
  23. var code = void 0;
  24. do {
  25. code = css.charCodeAt(next);
  26. if (wordDelimiters[code]) {
  27. return next - 1;
  28. } else if (code === t.backslash) {
  29. next = consumeEscape(css, next) + 1;
  30. } else {
  31. // All other characters are part of the word
  32. next++;
  33. }
  34. } while (next < css.length);
  35. return next - 1;
  36. }
  37. /**
  38. * Returns the last index of the escape sequence
  39. * @param {string} css The string in which the sequence begins
  40. * @param {number} start The index into the string where escape character (`\`) occurs.
  41. */
  42. function consumeEscape(css, start) {
  43. var next = start;
  44. var code = css.charCodeAt(next + 1);
  45. if (unescapable[code]) {
  46. // just consume the escape char
  47. } else if (hex[code]) {
  48. var hexDigits = 0;
  49. // consume up to 6 hex chars
  50. do {
  51. next++;
  52. hexDigits++;
  53. code = css.charCodeAt(next + 1);
  54. } while (hex[code] && hexDigits < 6);
  55. // if fewer than 6 hex chars, a trailing space ends the escape
  56. if (hexDigits < 6 && code === t.space) {
  57. next++;
  58. }
  59. } else {
  60. // the next char is part of the current word
  61. next++;
  62. }
  63. return next;
  64. }
  65. var FIELDS = exports.FIELDS = {
  66. TYPE: 0,
  67. START_LINE: 1,
  68. START_COL: 2,
  69. END_LINE: 3,
  70. END_COL: 4,
  71. START_POS: 5,
  72. END_POS: 6
  73. };
  74. function tokenize(input) {
  75. var tokens = [];
  76. var css = input.css.valueOf();
  77. var _css = css,
  78. length = _css.length;
  79. var offset = -1;
  80. var line = 1;
  81. var start = 0;
  82. var end = 0;
  83. var code = void 0,
  84. content = void 0,
  85. endColumn = void 0,
  86. endLine = void 0,
  87. escaped = void 0,
  88. escapePos = void 0,
  89. last = void 0,
  90. lines = void 0,
  91. next = void 0,
  92. nextLine = void 0,
  93. nextOffset = void 0,
  94. quote = void 0,
  95. tokenType = void 0;
  96. function unclosed(what, fix) {
  97. if (input.safe) {
  98. // fyi: this is never set to true.
  99. css += fix;
  100. next = css.length - 1;
  101. } else {
  102. throw input.error('Unclosed ' + what, line, start - offset, start);
  103. }
  104. }
  105. while (start < length) {
  106. code = css.charCodeAt(start);
  107. if (code === t.newline) {
  108. offset = start;
  109. line += 1;
  110. }
  111. switch (code) {
  112. case t.space:
  113. case t.tab:
  114. case t.newline:
  115. case t.cr:
  116. case t.feed:
  117. next = start;
  118. do {
  119. next += 1;
  120. code = css.charCodeAt(next);
  121. if (code === t.newline) {
  122. offset = next;
  123. line += 1;
  124. }
  125. } while (code === t.space || code === t.newline || code === t.tab || code === t.cr || code === t.feed);
  126. tokenType = t.space;
  127. endLine = line;
  128. endColumn = next - offset - 1;
  129. end = next;
  130. break;
  131. case t.plus:
  132. case t.greaterThan:
  133. case t.tilde:
  134. case t.pipe:
  135. next = start;
  136. do {
  137. next += 1;
  138. code = css.charCodeAt(next);
  139. } while (code === t.plus || code === t.greaterThan || code === t.tilde || code === t.pipe);
  140. tokenType = t.combinator;
  141. endLine = line;
  142. endColumn = start - offset;
  143. end = next;
  144. break;
  145. // Consume these characters as single tokens.
  146. case t.asterisk:
  147. case t.ampersand:
  148. case t.bang:
  149. case t.comma:
  150. case t.equals:
  151. case t.dollar:
  152. case t.caret:
  153. case t.openSquare:
  154. case t.closeSquare:
  155. case t.colon:
  156. case t.semicolon:
  157. case t.openParenthesis:
  158. case t.closeParenthesis:
  159. next = start;
  160. tokenType = code;
  161. endLine = line;
  162. endColumn = start - offset;
  163. end = next + 1;
  164. break;
  165. case t.singleQuote:
  166. case t.doubleQuote:
  167. quote = code === t.singleQuote ? "'" : '"';
  168. next = start;
  169. do {
  170. escaped = false;
  171. next = css.indexOf(quote, next + 1);
  172. if (next === -1) {
  173. unclosed('quote', quote);
  174. }
  175. escapePos = next;
  176. while (css.charCodeAt(escapePos - 1) === t.backslash) {
  177. escapePos -= 1;
  178. escaped = !escaped;
  179. }
  180. } while (escaped);
  181. tokenType = t.str;
  182. endLine = line;
  183. endColumn = start - offset;
  184. end = next + 1;
  185. break;
  186. default:
  187. if (code === t.slash && css.charCodeAt(start + 1) === t.asterisk) {
  188. next = css.indexOf('*/', start + 2) + 1;
  189. if (next === 0) {
  190. unclosed('comment', '*/');
  191. }
  192. content = css.slice(start, next + 1);
  193. lines = content.split('\n');
  194. last = lines.length - 1;
  195. if (last > 0) {
  196. nextLine = line + last;
  197. nextOffset = next - lines[last].length;
  198. } else {
  199. nextLine = line;
  200. nextOffset = offset;
  201. }
  202. tokenType = t.comment;
  203. line = nextLine;
  204. endLine = nextLine;
  205. endColumn = next - nextOffset;
  206. } else if (code === t.slash) {
  207. next = start;
  208. tokenType = code;
  209. endLine = line;
  210. endColumn = start - offset;
  211. end = next + 1;
  212. } else {
  213. next = consumeWord(css, start);
  214. tokenType = t.word;
  215. endLine = line;
  216. endColumn = next - offset;
  217. }
  218. end = next + 1;
  219. break;
  220. }
  221. // Ensure that the token structure remains consistent
  222. tokens.push([tokenType, // [0] Token type
  223. line, // [1] Starting line
  224. start - offset, // [2] Starting column
  225. endLine, // [3] Ending line
  226. endColumn, // [4] Ending column
  227. start, // [5] Start position / Source index
  228. end] // [6] End position
  229. );
  230. // Reset offset for the next token
  231. if (nextOffset) {
  232. offset = nextOffset;
  233. nextOffset = null;
  234. }
  235. start = end;
  236. }
  237. return tokens;
  238. }