TokenStreamBase.prototype = {
constructor: TokenStreamBase,
//-------------------------------------------------------------------------
//-------------------------------------------------------------------------
* Determines if the next token matches the given token type.
* If so, that token is consumed; if not, the token is placed
* back onto the token stream. You can pass in any number of
* token types and this will return true if any of the token
* @param {int|int[]} tokenTypes Either a single token type or an array of
* token types that the next token might be. If an array is passed,
* it's assumed that the token can be any of these.
* @param {variant} channel (Optional) The channel to read from. If not
* provided, reads from the default (unnamed) channel.
* @return {Boolean} True if the token type matches, false if not.
match: function(tokenTypes, channel) {
//always convert to an array, makes things easier
if (!(tokenTypes instanceof Array)) {
tokenTypes = [tokenTypes];
var tt = this.get(channel),
if (tt === tokenTypes[i++]) {
//no match found, put the token back
* Determines if the next token matches the given token type.
* If so, that token is consumed; if not, an error is thrown.
* @param {int|int[]} tokenTypes Either a single token type or an array of
* token types that the next token should be. If an array is passed,
* it's assumed that the token must be one of these.
mustMatch: function(tokenTypes) {
//always convert to an array, makes things easier
if (!(tokenTypes instanceof Array)) {
tokenTypes = [tokenTypes];
if (!this.match.apply(this, arguments)) {
throw new SyntaxError("Expected " + this._tokenData[tokenTypes[0]].name +
" at line " + token.startLine + ", col " + token.startCol + ".", token.startLine, token.startCol);
//-------------------------------------------------------------------------
//-------------------------------------------------------------------------
* Keeps reading from the token stream until either one of the specified
* token types is found or until the end of the input is reached.
* @param {int|int[]} tokenTypes Either a single token type or an array of
* token types that the next token should be. If an array is passed,
* it's assumed that the token must be one of these.
* @param {variant} channel (Optional) The channel to read from. If not
* provided, reads from the default (unnamed) channel.
advance: function(tokenTypes, channel) {
while (this.LA(0) !== 0 && !this.match(tokenTypes, channel)) {
* Consumes the next token from the token stream.
* @return {int} The token type of the token that was just consumed.
var tokenInfo = this._tokenData,
//check the lookahead buffer first
if (this._lt.length && this._ltIndex >= 0 && this._ltIndex < this._lt.length) {
this._token = this._lt[this._ltIndex++];
info = tokenInfo[this._token.type];
while ((info.channel !== undefined && channel !== info.channel) &&
this._ltIndex < this._lt.length) {
this._token = this._lt[this._ltIndex++];
info = tokenInfo[this._token.type];
if ((info.channel === undefined || channel === info.channel) &&
this._ltIndex <= this._lt.length) {
this._ltIndexCache.push(i);
//call token retriever method
token = this._getToken();
//if it should be hidden, don't save a token
if (token.type > -1 && !tokenInfo[token.type].hide) {
token.channel = tokenInfo[token.type].channel;
//save space that will be moved (must be done before array is truncated)
this._ltIndexCache.push(this._lt.length - this._ltIndex + i);
//keep the buffer under 5 items
if (this._lt.length > 5) {
//also keep the shift buffer under 5 items
if (this._ltIndexCache.length > 5) {
this._ltIndexCache.shift();
this._ltIndex = this._lt.length;
* Skip to the next token if:
* 1. The token type is marked as hidden.
* 2. The token type has a channel specified and it isn't the current channel.
info = tokenInfo[token.type];
(info.channel !== undefined && channel !== info.channel))) {
return this.get(channel);
* Looks ahead a certain number of tokens and returns the token type at
* that position. This will throw an error if you lookahead past the
* end of input, past the size of the lookahead buffer, or back past
* the first token in the lookahead buffer.
* @param {int} The index of the token type to retrieve. 0 for the
* current token, 1 for the next, -1 for the previous, etc.
* @return {int} The token type of the token in the given position.
//TODO: Store 5 somewhere
throw new Error("Too much lookahead.");
if (this._lt[this._ltIndex+index]) {
tt = this._lt[this._ltIndex+index].type;
throw new Error("Too much lookbehind.");
* Looks ahead a certain number of tokens and returns the token at
* that position. This will throw an error if you lookahead past the
* end of input, past the size of the lookahead buffer, or back past
* the first token in the lookahead buffer.
* @param {int} The index of the token type to retrieve. 0 for the
* current token, 1 for the next, -1 for the previous, etc.
* @return {Object} The token of the token in the given position.
//lookahead first to prime the token buffer
//now find the token, subtract one because _ltIndex is already at the next index
return this._lt[this._ltIndex+index-1];
* Returns the token type for the next token in the stream without
* @return {int} The token type of the next token in the stream.
* Returns the actual token object for the last consumed token.
* @return {Token} The token object for the last consumed token.
* Returns the name of the token for the given token type.
* @param {int} tokenType The type of token to get the name of.
* @return {String} The name of the token or "UNKNOWN_TOKEN" for any
tokenName: function(tokenType) {
if (tokenType < 0 || tokenType > this._tokenData.length) {
return this._tokenData[tokenType].name;
* Returns the token type value for the given token name.
* @param {String} tokenName The name of the token whose value should be returned.
* @return {int} The token type value for the given token name or -1
tokenType: function(tokenName) {
return this._tokenData[tokenName] || -1;
* Returns the last consumed token to the token stream.
//if (this._ltIndex > -1) {
if (this._ltIndexCache.length) {
this._ltIndex -= this._ltIndexCache.pop();//--;
this._token = this._lt[this._ltIndex - 1];
throw new Error("Too much lookahead.");
},{"./StringReader":24,"./SyntaxError":25}],28:[function(require,module,exports){
StringReader : require("./StringReader"),
SyntaxError : require("./SyntaxError"),
SyntaxUnit : require("./SyntaxUnit"),
EventTarget : require("./EventTarget"),
TokenStreamBase : require("./TokenStreamBase")
},{"./EventTarget":23,"./StringReader":24,"./SyntaxError":25,"./SyntaxUnit":26,"./TokenStreamBase":27}],"parserlib":[function(require,module,exports){
},{"./css":22,"./util":28}]},{},[]);
return require('parserlib');
var clone = (function() {
// maybe a reference error because no `Map`. Give it a dummy value that no
// value will ever be an instanceof.
nativeMap = function() {};
nativeSet = function() {};
nativePromise = function() {};
* Clones (copies) an Object using deep copying.
* This function supports circular references by default, but if you are certain
* there are no circular references in your object, you can save some CPU time
* by calling clone(obj, false).
* Caution: if `circular` is false and `parent` contains circular references,
* your program may enter an infinite loop and crash.
* @param `parent` - the object to be cloned
* @param `circular` - set to true if the object to be cloned may contain
* circular references. (optional - true by default)
* @param `depth` - set to a number if the object is only to be cloned to
* a particular depth. (optional - defaults to Infinity)
* @param `prototype` - sets the prototype to be used when cloning an object.
* (optional - defaults to parent prototype).
* @param `includeNonEnumerable` - set to true if the non-enumerable properties
* should be cloned as well. Non-enumerable properties on the prototype
* chain will be ignored. (optional - false by default)
function clone(parent, circular, depth, prototype, includeNonEnumerable) {
if (typeof circular === 'object') {
prototype = circular.prototype;
includeNonEnumerable = circular.includeNonEnumerable;
circular = circular.circular;
// maintain two arrays for circular references, where corresponding parents
// and children have the same index
var useBuffer = typeof Buffer != 'undefined';
if (typeof circular == 'undefined')
if (typeof depth == 'undefined')
// recurse this function so we don't reset allParents and allChildren
function _clone(parent, depth) {
// cloning null always returns null
if (typeof parent != 'object') {
if (parent instanceof nativeMap) {
} else if (parent instanceof nativeSet) {
} else if (parent instanceof nativePromise) {
child = new nativePromise(function (resolve, reject) {
parent.then(function(value) {
resolve(_clone(value, depth - 1));
reject(_clone(err, depth - 1));
} else if (clone.__isArray(parent)) {
} else if (clone.__isRegExp(parent)) {
child = new RegExp(parent.source, __getRegExpFlags(parent));
if (parent.lastIndex) child.lastIndex = parent.lastIndex;
} else if (clone.__isDate(parent)) {
child = new Date(parent.getTime());
} else if (useBuffer && Buffer.isBuffer(parent)) {
child = new Buffer(parent.length);
} else if (parent instanceof Error) {
child = Object.create(parent);
if (typeof prototype == 'undefined') {
proto = Object.getPrototypeOf(parent);
child = Object.create(proto);
child = Object.create(prototype);
var index = allParents.indexOf(parent);
return allChildren[index];
if (parent instanceof nativeMap) {
var keyIterator = parent.keys();
var next = keyIterator.next();
var keyChild = _clone(next.value, depth - 1);
var valueChild = _clone(parent.get(next.value), depth - 1);
child.set(keyChild, valueChild);
if (parent instanceof nativeSet) {
var iterator = parent.keys();
var next = iterator.next();
var entryChild = _clone(next.value, depth - 1);
attrs = Object.getOwnPropertyDescriptor(proto, i);
if (attrs && attrs.set == null) {
child[i] = _clone(parent[i], depth - 1);
if (Object.getOwnPropertySymbols) {
var symbols = Object.getOwnPropertySymbols(parent);
for (var i = 0; i < symbols.length; i++) {
// Don't need to worry about cloning a symbol because it is a primitive,
// like a number or string.
It is recommended that you Edit text format, this type of Fix handles quite a lot in one request
