/**

  * Copyright (c) 2008-2012, http://www.snakeyaml.org

  *

  * Licensed under the Apache License, Version 2.0 (the "License");

  * you may not use this file except in compliance with the License.

  * You may obtain a copy of the License at

  *

  *     http://www.apache.org/licenses/LICENSE-2.0

  *

  * Unless required by applicable law or agreed to in writing, software

  * distributed under the License is distributed on an "AS IS" BASIS,

  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

  * See the License for the specific language governing permissions and

  * limitations under the License.

  */

 package org.yaml.snakeyaml.scanner;

 import java.nio.ByteBuffer;

 import java.nio.charset.CharacterCodingException;

 import java.util.ArrayList;

 import java.util.HashMap;

 import java.util.Iterator;

 import java.util.LinkedHashMap;

 import java.util.List;

 import java.util.Map;

 import java.util.regex.Pattern;

 import org.yaml.snakeyaml.error.Mark;

 import org.yaml.snakeyaml.error.YAMLException;

 import org.yaml.snakeyaml.reader.StreamReader;

 import org.yaml.snakeyaml.tokens.AliasToken;

 import org.yaml.snakeyaml.tokens.AnchorToken;

 import org.yaml.snakeyaml.tokens.BlockEndToken;

 import org.yaml.snakeyaml.tokens.BlockEntryToken;

 import org.yaml.snakeyaml.tokens.BlockMappingStartToken;

 import org.yaml.snakeyaml.tokens.BlockSequenceStartToken;

 import org.yaml.snakeyaml.tokens.DirectiveToken;

 import org.yaml.snakeyaml.tokens.DocumentEndToken;

 import org.yaml.snakeyaml.tokens.DocumentStartToken;

 import org.yaml.snakeyaml.tokens.FlowEntryToken;

 import org.yaml.snakeyaml.tokens.FlowMappingEndToken;

 import org.yaml.snakeyaml.tokens.FlowMappingStartToken;

 import org.yaml.snakeyaml.tokens.FlowSequenceEndToken;

 import org.yaml.snakeyaml.tokens.FlowSequenceStartToken;

 import org.yaml.snakeyaml.tokens.KeyToken;

 import org.yaml.snakeyaml.tokens.ScalarToken;

 import org.yaml.snakeyaml.tokens.StreamEndToken;

 import org.yaml.snakeyaml.tokens.StreamStartToken;

 import org.yaml.snakeyaml.tokens.TagToken;

 import org.yaml.snakeyaml.tokens.TagTuple;

 import org.yaml.snakeyaml.tokens.Token;

 import org.yaml.snakeyaml.tokens.ValueToken;

 import org.yaml.snakeyaml.util.ArrayStack;

 import org.yaml.snakeyaml.util.UriEncoder;

 /**

  * <pre>

  * Scanner produces tokens of the following types:

  * STREAM-START

  * STREAM-END

  * DIRECTIVE(name, value)

  * DOCUMENT-START

  * DOCUMENT-END

  * BLOCK-SEQUENCE-START

  * BLOCK-MAPPING-START

  * BLOCK-END

  * FLOW-SEQUENCE-START

  * FLOW-MAPPING-START

  * FLOW-SEQUENCE-END

  * FLOW-MAPPING-END

  * BLOCK-ENTRY

  * FLOW-ENTRY

  * KEY

  * VALUE

  * ALIAS(value)

  * ANCHOR(value)

  * TAG(value)

  * SCALAR(value, plain, style)

  * Read comments in the Scanner code for more details.

  * </pre>

  */

 public final class ScannerImpl implements Scanner {

     /**

      * A regular expression matching characters which are not in the hexadecimal

      * set (0-9, A-F, a-f).

      */

     private final static Pattern NOT_HEXA = Pattern.compile("[^0-9A-Fa-f]");

     /**

      * A mapping from an escaped character in the input stream to the character

      * that they should be replaced with.

      * 

      * YAML defines several common and a few uncommon escape sequences.

      * 

      * @see <a href="http://www.yaml.org/spec/current.html#id2517668">4.1.6.

      *      Escape Sequences</a>

      */

     public final static Map<Character, String> ESCAPE_REPLACEMENTS = new HashMap<Character, String>();

     /**

      * A mapping from a character to a number of bytes to read-ahead for that

      * escape sequence. These escape sequences are used to handle unicode

      * escaping in the following formats, where H is a hexadecimal character:

      * 

      * <pre>

      * \xHH         : escaped 8-bit Unicode character

      * \uHHHH       : escaped 16-bit Unicode character

      * \UHHHHHHHH   : escaped 32-bit Unicode character

      * </pre>

      * 

      * @see <a href="http://yaml.org/spec/1.1/current.html#id872840">5.6. Escape

      *      Sequences</a>

      */

     public final static Map<Character, Integer> ESCAPE_CODES = new HashMap<Character, Integer>();

     static {

         // ASCII null

         ESCAPE_REPLACEMENTS.put(Character.valueOf('0'), "\0");

         // ASCII bell

         ESCAPE_REPLACEMENTS.put(Character.valueOf('a'), "\u0007");

         // ASCII backspace

         ESCAPE_REPLACEMENTS.put(Character.valueOf('b'), "\u0008");

         // ASCII horizontal tab

         ESCAPE_REPLACEMENTS.put(Character.valueOf('t'), "\u0009");

         // ASCII newline (line feed; \n maps to 0x0A)

         ESCAPE_REPLACEMENTS.put(Character.valueOf('n'), "\n");

         // ASCII vertical tab

         ESCAPE_REPLACEMENTS.put(Character.valueOf('v'), "\u000B");

         // ASCII form-feed

         ESCAPE_REPLACEMENTS.put(Character.valueOf('f'), "\u000C");

         // carriage-return (\r maps to 0x0D)

         ESCAPE_REPLACEMENTS.put(Character.valueOf('r'), "\r");

         // ASCII escape character (Esc)

         ESCAPE_REPLACEMENTS.put(Character.valueOf('e'), "\u001B");

         // ASCII space

         ESCAPE_REPLACEMENTS.put(Character.valueOf(' '), "\u0020");

         // ASCII double-quote

         ESCAPE_REPLACEMENTS.put(Character.valueOf('"'), "\"");

         // ASCII backslash

         ESCAPE_REPLACEMENTS.put(Character.valueOf('\\'), "\\");

         // Unicode next line

         ESCAPE_REPLACEMENTS.put(Character.valueOf('N'), "\u0085");

         // Unicode non-breaking-space

         ESCAPE_REPLACEMENTS.put(Character.valueOf('_'), "\u00A0");

         // Unicode line-separator

         ESCAPE_REPLACEMENTS.put(Character.valueOf('L'), "\u2028");

         // Unicode paragraph separator

         ESCAPE_REPLACEMENTS.put(Character.valueOf('P'), "\u2029");

         // 8-bit Unicode

         ESCAPE_CODES.put(Character.valueOf('x'), 2);

         // 16-bit Unicode

         ESCAPE_CODES.put(Character.valueOf('u'), 4);

         // 32-bit Unicode (Supplementary characters are supported)

         ESCAPE_CODES.put(Character.valueOf('U'), 8);

     }

     private final StreamReader reader;

     // Had we reached the end of the stream?

     private boolean done = false;

     // The number of unclosed '{' and '['. `flow_level == 0` means block

     // context.

     private int flowLevel = 0;

     // List of processed tokens that are not yet emitted.

     private List<Token> tokens;

     // Number of tokens that were emitted through the `get_token` method.

     private int tokensTaken = 0;

     // The current indentation level.

     private int indent = -1;

     // Past indentation levels.

     private ArrayStack<Integer> indents;

     // Variables related to simple keys treatment. See PyYAML.

     /**

      * <pre>

      * A simple key is a key that is not denoted by the '?' indicator.

      * Example of simple keys:

      *   ---

      *   block simple key: value

      *   ? not a simple key:

      *   : { flow simple key: value }

      * We emit the KEY token before all keys, so when we find a potential

      * simple key, we try to locate the corresponding ':' indicator.

      * Simple keys should be limited to a single line and 1024 characters.

      * 

      * Can a simple key start at the current position? A simple key may

      * start:

      * - at the beginning of the line, not counting indentation spaces

      *       (in block context),

      * - after '{', '[', ',' (in the flow context),

      * - after '?', ':', '-' (in the block context).

      * In the block context, this flag also signifies if a block collection

      * may start at the current position.

      * </pre>

      */

     private boolean allowSimpleKey = true;

     /*

      * Keep track of possible simple keys. This is a dictionary. The key is

      * `flow_level`; there can be no more that one possible simple key for each

      * level. The value is a SimpleKey record: (token_number, required, index,

      * line, column, mark) A simple key may start with ALIAS, ANCHOR, TAG,

      * SCALAR(flow), '[', or '{' tokens.

      */

     private Map<Integer, SimpleKey> possibleSimpleKeys;

     public ScannerImpl(StreamReader reader) {

         this.reader = reader;

         this.tokens = new ArrayList<Token>(100);

         this.indents = new ArrayStack<Integer>(10);

         // The order in possibleSimpleKeys is kept for nextPossibleSimpleKey()

         this.possibleSimpleKeys = new LinkedHashMap<Integer, SimpleKey>();

         fetchStreamStart();// Add the STREAM-START token.

     }

     /**

      * Check whether the next token is one of the given types.

      */

     public boolean checkToken(Token.ID... choices) {

         while (needMoreTokens()) {

             fetchMoreTokens();

         }

         if (!this.tokens.isEmpty()) {

             if (choices.length == 0) {

                 return true;

             }

             // since profiler puts this method on top (it is used a lot), we

             // should not use 'foreach' here because of the performance reasons

             Token.ID first = this.tokens.get(0).getTokenId();

             for (int i = 0; i < choices.length; i++) {

                 if (first == choices[i]) {

                     return true;

                 }

             }

         }

         return false;

     }

     /**

      * Return the next token, but do not delete it from the queue.

      */

     public Token peekToken() {

         while (needMoreTokens()) {

             fetchMoreTokens();

         }

         return this.tokens.get(0);

     }

     /**

      * Return the next token, removing it from the queue.

      */

     public Token getToken() {

         if (!this.tokens.isEmpty()) {

             this.tokensTaken++;

             return this.tokens.remove(0);

         }

         return null;

     }

     // Private methods.

     /**

      * Returns true if more tokens should be scanned.

      */

     private boolean needMoreTokens() {

         // If we are done, we do not require more tokens.

         if (this.done) {

             return false;

         }

         // If we aren't done, but we have no tokens, we need to scan more.

         if (this.tokens.isEmpty()) {

             return true;

         }

         // The current token may be a potential simple key, so we

         // need to look further.

         stalePossibleSimpleKeys();

         return nextPossibleSimpleKey() == this.tokensTaken;

     }

     /**

      * Fetch one or more tokens from the StreamReader.

      */

     private void fetchMoreTokens() {

         // Eat whitespaces and comments until we reach the next token.

         scanToNextToken();

         // Remove obsolete possible simple keys.

         stalePossibleSimpleKeys();

         // Compare the current indentation and column. It may add some tokens

         // and decrease the current indentation level.

         unwindIndent(reader.getColumn());

         // Peek the next character, to decide what the next group of tokens

         // will look like.

         char ch = reader.peek();

         switch (ch) {

         case '\0':

             // Is it the end of stream?

             fetchStreamEnd();

             return;

         case '%':

             // Is it a directive?

             if (checkDirective()) {

                 fetchDirective();

                 return;

             }

             break;

         case '-':

             // Is it the document start?

             if (checkDocumentStart()) {

                 fetchDocumentStart();

                 return;

                 // Is it the block entry indicator?

             } else if (checkBlockEntry()) {

                 fetchBlockEntry();

                 return;

             }

             break;

         case '.':

             // Is it the document end?

             if (checkDocumentEnd()) {

                 fetchDocumentEnd();

                 return;

             }

             break;

         // TODO support for BOM within a stream. (not implemented in PyYAML)

         case '[':

             // Is it the flow sequence start indicator?

             fetchFlowSequenceStart();

             return;

         case '{':

             // Is it the flow mapping start indicator?

             fetchFlowMappingStart();

             return;

         case ']':

             // Is it the flow sequence end indicator?

             fetchFlowSequenceEnd();

             return;

         case '}':

             // Is it the flow mapping end indicator?

             fetchFlowMappingEnd();

             return;

         case ',':

             // Is it the flow entry indicator?

             fetchFlowEntry();

             return;

             // see block entry indicator above

         case '?':

             // Is it the key indicator?

             if (checkKey()) {

                 fetchKey();

                 return;

             }

             break;

         case ':':

             // Is it the value indicator?

             if (checkValue()) {

                 fetchValue();

                 return;

             }

             break;

         case '*':

             // Is it an alias?

             fetchAlias();

             return;

         case '&':

             // Is it an anchor?

             fetchAnchor();

             return;

         case '!':

             // Is it a tag?

             fetchTag();

             return;

         case '|':

             // Is it a literal scalar?

             if (this.flowLevel == 0) {

                 fetchLiteral();

                 return;

             }

             break;

         case '>':

             // Is it a folded scalar?

             if (this.flowLevel == 0) {

                 fetchFolded();

                 return;

             }

             break;

         case '\'':

             // Is it a single quoted scalar?

             fetchSingle();

             return;

         case '"':

             // Is it a double quoted scalar?

             fetchDouble();

             return;

         }

         // It must be a plain scalar then.

         if (checkPlain()) {

             fetchPlain();

             return;

         }

         // No? It's an error. Let's produce a nice error message.We do this by

         // converting escaped characters into their escape sequences. This is a

         // backwards use of the ESCAPE_REPLACEMENTS map.

         String chRepresentation = String.valueOf(ch);

         for (Character s : ESCAPE_REPLACEMENTS.keySet()) {

             String v = ESCAPE_REPLACEMENTS.get(s);

             if (v.equals(chRepresentation)) {

                 chRepresentation = "\\" + s;// ' ' -> '\t'

                 break;

             }

         }

         throw new ScannerException("while scanning for the next token", null, "found character "

                 + ch + "'" + chRepresentation + "' that cannot start any token", reader.getMark());

     }

     // Simple keys treatment.

     /**

      * Return the number of the nearest possible simple key. Actually we don't

      * need to loop through the whole dictionary.

      */

     private int nextPossibleSimpleKey() {

         /*

          * the implementation is not as in PyYAML. Because

          * this.possibleSimpleKeys is ordered we can simply take the first key

          */

         if (!this.possibleSimpleKeys.isEmpty()) {

             return this.possibleSimpleKeys.values().iterator().next().getTokenNumber();

         }

         return -1;

     }

     /**

      * <pre>

      * Remove entries that are no longer possible simple keys. According to

      * the YAML specification, simple keys

      * - should be limited to a single line,

      * - should be no longer than 1024 characters.

      * Disabling this procedure will allow simple keys of any length and

      * height (may cause problems if indentation is broken though).

      * </pre>

      */

     private void stalePossibleSimpleKeys() {

         if (!this.possibleSimpleKeys.isEmpty()) {

             for (Iterator<SimpleKey> iterator = this.possibleSimpleKeys.values().iterator(); iterator

                     .hasNext();) {

                 SimpleKey key = iterator.next();

                 if ((key.getLine() != reader.getLine())

                         || (reader.getIndex() - key.getIndex() > 1024)) {

                     // If the key is not on the same line as the current

                     // position OR the difference in column between the token

                     // start and the current position is more than the maximum

                     // simple key length, then this cannot be a simple key.

                     if (key.isRequired()) {

                         // If the key was required, this implies an error

                         // condition.

                         throw new ScannerException("while scanning a simple key", key.getMark(),

                                 "could not found expected ':'", reader.getMark());

                     }

                     iterator.remove();

                 }

             }

         }

     }

     /**

      * The next token may start a simple key. We check if it's possible and save

      * its position. This function is called for ALIAS, ANCHOR, TAG,

      * SCALAR(flow), '[', and '{'.

      */

     private void savePossibleSimpleKey() {

         // The next token may start a simple key. We check if it's possible

         // and save its position. This function is called for

         // ALIAS, ANCHOR, TAG, SCALAR(flow), '[', and '{'.

         // Check if a simple key is required at the current position.

         // A simple key is required if this position is the root flowLevel, AND

         // the current indentation level is the same as the last indent-level.

         boolean required = ((this.flowLevel == 0) && (this.indent == this.reader.getColumn()));

         if (allowSimpleKey || !required) {

             // A simple key is required only if it is the first token in the

             // current line. Therefore it is always allowed.

         } else {

             throw new YAMLException(

                     "A simple key is required only if it is the first token in the current line");

         }

         // The next token might be a simple key. Let's save it's number and

         // position.

         if (this.allowSimpleKey) {

             removePossibleSimpleKey();

             int tokenNumber = this.tokensTaken + this.tokens.size();

             SimpleKey key = new SimpleKey(tokenNumber, required, reader.getIndex(),

                     reader.getLine(), this.reader.getColumn(), this.reader.getMark());

             this.possibleSimpleKeys.put(this.flowLevel, key);

         }

     }

     /**

      * Remove the saved possible key position at the current flow level.

      */

     private void removePossibleSimpleKey() {

         SimpleKey key = possibleSimpleKeys.remove(flowLevel);

         if (key != null && key.isRequired()) {

             throw new ScannerException("while scanning a simple key", key.getMark(),

                     "could not found expected ':'", reader.getMark());

         }

     }

     // Indentation functions.

     /**

      * * Handle implicitly ending multiple levels of block nodes by decreased

      * indentation. This function becomes important on lines 4 and 7 of this

      * example:

      * 

      * <pre>

      * 1) book one:

      * 2)   part one:

      * 3)     chapter one

      * 4)   part two:

      * 5)     chapter one

      * 6)     chapter two

      * 7) book two:

      * </pre>

      * 

      * In flow context, tokens should respect indentation. Actually the

      * condition should be `self.indent >= column` according to the spec. But

      * this condition will prohibit intuitively correct constructions such as

      * key : { } </pre>

      */

     private void unwindIndent(int col) {

         // In the flow context, indentation is ignored. We make the scanner less

         // restrictive then specification requires.

         if (this.flowLevel != 0) {

             return;

         }

         // In block context, we may need to issue the BLOCK-END tokens.

         while (this.indent > col) {

             Mark mark = reader.getMark();

             this.indent = this.indents.pop();

             this.tokens.add(new BlockEndToken(mark, mark));

         }

     }

     /**

      * Check if we need to increase indentation.

      */

     private boolean addIndent(int column) {

         if (this.indent < column) {

             this.indents.push(this.indent);

             this.indent = column;

             return true;

         }

         return false;

     }

     // Fetchers.

     /**

      * We always add STREAM-START as the first token and STREAM-END as the last

      * token.

      */

     private void fetchStreamStart() {

         // Read the token.

         Mark mark = reader.getMark();

         // Add STREAM-START.

         Token token = new StreamStartToken(mark, mark);

         this.tokens.add(token);

     }

     private void fetchStreamEnd() {

         // Set the current intendation to -1.

         unwindIndent(-1);

         // Reset simple keys.

         removePossibleSimpleKey();

         this.allowSimpleKey = false;

         this.possibleSimpleKeys.clear();

         // Read the token.

         Mark mark = reader.getMark();

         // Add STREAM-END.

         Token token = new StreamEndToken(mark, mark);

         this.tokens.add(token);

         // The stream is finished.

         this.done = true;

     }

     /**

      * Fetch a YAML directive. Directives are presentation details that are

      * interpreted as instructions to the processor. YAML defines two kinds of

      * directives, YAML and TAG; all other types are reserved for future use.

      * 

      * @see http://www.yaml.org/spec/1.1/#id864824

      */

     private void fetchDirective() {

         // Set the current intendation to -1.

         unwindIndent(-1);

         // Reset simple keys.

         removePossibleSimpleKey();

         this.allowSimpleKey = false;

         // Scan and add DIRECTIVE.

         Token tok = scanDirective();

         this.tokens.add(tok);

     }

     /**

      * Fetch a document-start token ("---").

      */

     private void fetchDocumentStart() {

         fetchDocumentIndicator(true);

     }

     /**

      * Fetch a document-end token ("...").

      */

     private void fetchDocumentEnd() {

         fetchDocumentIndicator(false);

     }

     /**

      * Fetch a document indicator, either "---" for "document-start", or else

      * "..." for "document-end. The type is chosen by the given boolean.

      */

     private void fetchDocumentIndicator(boolean isDocumentStart) {

         // Set the current intendation to -1.

         unwindIndent(-1);

         // Reset simple keys. Note that there could not be a block collection

         // after '---'.

         removePossibleSimpleKey();

         this.allowSimpleKey = false;

         // Add DOCUMENT-START or DOCUMENT-END.

         Mark startMark = reader.getMark();

         reader.forward(3);

         Mark endMark = reader.getMark();

         Token token;

         if (isDocumentStart) {

             token = new DocumentStartToken(startMark, endMark);

         } else {

             token = new DocumentEndToken(startMark, endMark);

         }

         this.tokens.add(token);

     }

     private void fetchFlowSequenceStart() {

         fetchFlowCollectionStart(false);

     }

     private void fetchFlowMappingStart() {

         fetchFlowCollectionStart(true);

     }

     /**

      * Fetch a flow-style collection start, which is either a sequence or a

      * mapping. The type is determined by the given boolean.

      * 

      * A flow-style collection is in a format similar to JSON. Sequences are

      * started by '[' and ended by ']'; mappings are started by '{' and ended by

      * '}'.

      * 

      * @see http://www.yaml.org/spec/1.1/#id863975

      * 

      * @param isMappingStart

      */

     private void fetchFlowCollectionStart(boolean isMappingStart) {

         // '[' and '{' may start a simple key.

         savePossibleSimpleKey();

         // Increase the flow level.

         this.flowLevel++;

         // Simple keys are allowed after '[' and '{'.

         this.allowSimpleKey = true;

         // Add FLOW-SEQUENCE-START or FLOW-MAPPING-START.

         Mark startMark = reader.getMark();

         reader.forward(1);

         Mark endMark = reader.getMark();

         Token token;

         if (isMappingStart) {

             token = new FlowMappingStartToken(startMark, endMark);

         } else {

             token = new FlowSequenceStartToken(startMark, endMark);

         }

         this.tokens.add(token);

     }

     private void fetchFlowSequenceEnd() {

         fetchFlowCollectionEnd(false);

     }

     private void fetchFlowMappingEnd() {

         fetchFlowCollectionEnd(true);

     }

     /**

      * Fetch a flow-style collection end, which is either a sequence or a

      * mapping. The type is determined by the given boolean.

      * 

      * A flow-style collection is in a format similar to JSON. Sequences are

      * started by '[' and ended by ']'; mappings are started by '{' and ended by

      * '}'.

      * 

      * @see http://www.yaml.org/spec/1.1/#id863975

      */

     private void fetchFlowCollectionEnd(boolean isMappingEnd) {

         // Reset possible simple key on the current level.

         removePossibleSimpleKey();

         // Decrease the flow level.

         this.flowLevel--;

         // No simple keys after ']' or '}'.

         this.allowSimpleKey = false;

         // Add FLOW-SEQUENCE-END or FLOW-MAPPING-END.

         Mark startMark = reader.getMark();

         reader.forward();

         Mark endMark = reader.getMark();

         Token token;

         if (isMappingEnd) {

             token = new FlowMappingEndToken(startMark, endMark);

         } else {

             token = new FlowSequenceEndToken(startMark, endMark);

         }

         this.tokens.add(token);

     }

     /**

      * Fetch an entry in the flow style. Flow-style entries occur either

      * immediately after the start of a collection, or else after a comma.

      * 

      * @see http://www.yaml.org/spec/1.1/#id863975

      */

     private void fetchFlowEntry() {

         // Simple keys are allowed after ','.

         this.allowSimpleKey = true;

         // Reset possible simple key on the current level.

         removePossibleSimpleKey();

         // Add FLOW-ENTRY.

         Mark startMark = reader.getMark();

         reader.forward();

         Mark endMark = reader.getMark();

         Token token = new FlowEntryToken(startMark, endMark);

         this.tokens.add(token);

     }

     /**

      * Fetch an entry in the block style.

      * 

      * @see http://www.yaml.org/spec/1.1/#id863975

      */

     private void fetchBlockEntry() {

         // Block context needs additional checks.

         if (this.flowLevel == 0) {

             // Are we allowed to start a new entry?

             if (!this.allowSimpleKey) {

                 throw new ScannerException(null, null, "sequence entries are not allowed here",

                         reader.getMark());

             }

             // We may need to add BLOCK-SEQUENCE-START.

             if (addIndent(this.reader.getColumn())) {

                 Mark mark = reader.getMark();

                 this.tokens.add(new BlockSequenceStartToken(mark, mark));

             }

         } else {

             // It's an error for the block entry to occur in the flow

             // context,but we let the parser detect this.

         }

         // Simple keys are allowed after '-'.

         this.allowSimpleKey = true;

         // Reset possible simple key on the current level.

         removePossibleSimpleKey();

         // Add BLOCK-ENTRY.

         Mark startMark = reader.getMark();

         reader.forward();

         Mark endMark = reader.getMark();

         Token token = new BlockEntryToken(startMark, endMark);

         this.tokens.add(token);

     }

     /**

      * Fetch a key in a block-style mapping.

      * 

      * @see http://www.yaml.org/spec/1.1/#id863975

      */

     private void fetchKey() {

         // Block context needs additional checks.

         if (this.flowLevel == 0) {

             // Are we allowed to start a key (not necessary a simple)?

             if (!this.allowSimpleKey) {

                 throw new ScannerException(null, null, "mapping keys are not allowed here",

                         reader.getMark());

             }

             // We may need to add BLOCK-MAPPING-START.

             if (addIndent(this.reader.getColumn())) {

                 Mark mark = reader.getMark();

                 this.tokens.add(new BlockMappingStartToken(mark, mark));

             }

         }

         // Simple keys are allowed after '?' in the block context.

         this.allowSimpleKey = this.flowLevel == 0;

         // Reset possible simple key on the current level.

         removePossibleSimpleKey();

         // Add KEY.

         Mark startMark = reader.getMark();

         reader.forward();

         Mark endMark = reader.getMark();

         Token token = new KeyToken(startMark, endMark);

         this.tokens.add(token);

     }

     /**

      * Fetch a value in a block-style mapping.

      * 

      * @see http://www.yaml.org/spec/1.1/#id863975

      */

     private void fetchValue() {

         // Do we determine a simple key?

         SimpleKey key = this.possibleSimpleKeys.remove(this.flowLevel);

         if (key != null) {

             // Add KEY.

             this.tokens.add(key.getTokenNumber() - this.tokensTaken, new KeyToken(key.getMark(),

                     key.getMark()));

             // If this key starts a new block mapping, we need to add

             // BLOCK-MAPPING-START.

             if (this.flowLevel == 0) {

                 if (addIndent(key.getColumn())) {

                     this.tokens.add(key.getTokenNumber() - this.tokensTaken,

                             new BlockMappingStartToken(key.getMark(), key.getMark()));

                 }

             }

             // There cannot be two simple keys one after another.

             this.allowSimpleKey = false;

         } else {// It must be a part of a complex key.

             // Block context needs additional checks.Do we really need them?

             // They

             // will be catched by the parser anyway.)

             if (this.flowLevel == 0) {

                 // We are allowed to start a complex value if and only if we can

                 // start a simple key.

                 if (!this.allowSimpleKey) {

                     throw new ScannerException(null, null, "mapping values are not allowed here",

                             reader.getMark());

                 }

             }

             // If this value starts a new block mapping, we need to add

             // BLOCK-MAPPING-START. It will be detected as an error later by

             // the parser.

             if (flowLevel == 0) {

                 if (addIndent(reader.getColumn())) {

                     Mark mark = reader.getMark();

                     this.tokens.add(new BlockMappingStartToken(mark, mark));

                 }

             }

             // Simple keys are allowed after ':' in the block context.

             allowSimpleKey = (flowLevel == 0);

             // Reset possible simple key on the current level.

             removePossibleSimpleKey();

         }

         // Add VALUE.

         Mark startMark = reader.getMark();

         reader.forward();

         Mark endMark = reader.getMark();

         Token token = new ValueToken(startMark, endMark);

         this.tokens.add(token);

     }

     /**

      * Fetch an alias, which is a reference to an anchor. Aliases take the

      * format:

      * 

      * <pre>

      * *(anchor name)

      * </pre>

      * 

      * @see http://www.yaml.org/spec/1.1/#id863390

      */

     private void fetchAlias() {

         // ALIAS could be a simple key.

         savePossibleSimpleKey();

         // No simple keys after ALIAS.

         this.allowSimpleKey = false;

         // Scan and add ALIAS.

         Token tok = scanAnchor(false);

         this.tokens.add(tok);

     }

     /**

      * Fetch an anchor. Anchors take the form:

      * 

      * <pre>

      * &(anchor name)

      * </pre>

      * 

      * @see http://www.yaml.org/spec/1.1/#id863390

      */

     private void fetchAnchor() {

         // ANCHOR could start a simple key.

         savePossibleSimpleKey();

         // No simple keys after ANCHOR.

         this.allowSimpleKey = false;

         // Scan and add ANCHOR.

         Token tok = scanAnchor(true);

         this.tokens.add(tok);

     }

     /**

      * Fetch a tag. Tags take a complex form.

      * 

      * @see http://www.yaml.org/spec/1.1/#id861700

      */

     private void fetchTag() {

         // TAG could start a simple key.

         savePossibleSimpleKey();

         // No simple keys after TAG.

         this.allowSimpleKey = false;

         // Scan and add TAG.

         Token tok = scanTag();

         this.tokens.add(tok);

     }

     /**

      * Fetch a literal scalar, denoted with a vertical-bar. This is the type

      * best used for source code and other content, such as binary data, which

      * must be included verbatim.

      * 

      * @see http://www.yaml.org/spec/1.1/#id863975

      */

     private void fetchLiteral() {

         fetchBlockScalar('|');

     }

     /**

      * Fetch a folded scalar, denoted with a greater-than sign. This is the type

      * best used for long content, such as the text of a chapter or description.

      * 

      * @see http://www.yaml.org/spec/1.1/#id863975

      */

     private void fetchFolded() {

         fetchBlockScalar('>');

     }

     /**

      * Fetch a block scalar (literal or folded).

      * 

      * @see http://www.yaml.org/spec/1.1/#id863975

      * 

      * @param style

      */

     private void fetchBlockScalar(char style) {

         // A simple key may follow a block scalar.

         this.allowSimpleKey = true;

         // Reset possible simple key on the current level.

         removePossibleSimpleKey();

         // Scan and add SCALAR.

         Token tok = scanBlockScalar(style);

         this.tokens.add(tok);

     }

     /**

      * Fetch a single-quoted (') scalar.

      */

     private void fetchSingle() {

         fetchFlowScalar('\'');

     }

     /**

      * Fetch a double-quoted (") scalar.

      */

     private void fetchDouble() {

         fetchFlowScalar('"');

     }

     /**

      * Fetch a flow scalar (single- or double-quoted).

      * 

      * @see http://www.yaml.org/spec/1.1/#id863975

      * 

      * @param style

      */

     private void fetchFlowScalar(char style) {

         // A flow scalar could be a simple key.

         savePossibleSimpleKey();

         // No simple keys after flow scalars.

         this.allowSimpleKey = false;

         // Scan and add SCALAR.

         Token tok = scanFlowScalar(style);

         this.tokens.add(tok);

     }

     /**

      * Fetch a plain scalar.

      */

     private void fetchPlain() {

         // A plain scalar could be a simple key.

         savePossibleSimpleKey();

         // No simple keys after plain scalars. But note that `scan_plain` will

         // change this flag if the scan is finished at the beginning of the

         // line.

         this.allowSimpleKey = false;

         // Scan and add SCALAR. May change `allow_simple_key`.

         Token tok = scanPlain();

         this.tokens.add(tok);

     }

     // Checkers.

     /**

      * Returns true if the next thing on the reader is a directive, given that

      * the leading '%' has already been checked.

      * 

      * @see http://www.yaml.org/spec/1.1/#id864824

      */

     private boolean checkDirective() {

         // DIRECTIVE: ^ '%' ...

         // The '%' indicator is already checked.

         return reader.getColumn() == 0;

     }

     /**

      * Returns true if the next thing on the reader is a document-start ("---").

      * A document-start is always followed immediately by a new line.

      */

     private boolean checkDocumentStart() {

         // DOCUMENT-START: ^ '---' (' '|'\n')

         if (reader.getColumn() == 0) {

             if ("---".equals(reader.prefix(3)) && Constant.NULL_BL_T_LINEBR.has(reader.peek(3))) {

                 return true;

             }

         }

         return false;

     }

     /**

      * Returns true if the next thing on the reader is a document-end ("..."). A

      * document-end is always followed immediately by a new line.

      */

     private boolean checkDocumentEnd() {

         // DOCUMENT-END: ^ '...' (' '|'\n')

         if (reader.getColumn() == 0) {

             if ("...".equals(reader.prefix(3)) && Constant.NULL_BL_T_LINEBR.has(reader.peek(3))) {

                 return true;

             }

         }

         return false;

     }

     /**

      * Returns true if the next thing on the reader is a block token.

      */

     private boolean checkBlockEntry() {

         // BLOCK-ENTRY: '-' (' '|'\n')

         return Constant.NULL_BL_T_LINEBR.has(reader.peek(1));

     }

     /**

      * Returns true if the next thing on the reader is a key token.

      */

     private boolean checkKey() {

         // KEY(flow context): '?'

         if (this.flowLevel != 0) {

             return true;

         } else {

             // KEY(block context): '?' (' '|'\n')

             return Constant.NULL_BL_T_LINEBR.has(reader.peek(1));

         }

     }

     /**

      * Returns true if the next thing on the reader is a value token.

      */

     private boolean checkValue() {

         // VALUE(flow context): ':'

         if (flowLevel != 0) {

             return true;

         } else {

             // VALUE(block context): ':' (' '|'\n')

             return Constant.NULL_BL_T_LINEBR.has(reader.peek(1));

         }

     }

     /**

      * Returns true if the next thing on the reader is a plain token.

      */

     private boolean checkPlain() {

         /**

          * <pre>

          * A plain scalar may start with any non-space character except:

          *   '-', '?', ':', ',', '[', ']', '{', '}',

          *   '#', '&', '*', '!', '|', '>', '\'', '\"',

          *   '%', '@', '`'.

          * 

          * It may also start with

          *   '-', '?', ':'

          * if it is followed by a non-space character.

          * 

          * Note that we limit the last rule to the block context (except the

          * '-' character) because we want the flow context to be space

          * independent.

          * </pre>

          */

         char ch = reader.peek();

         // If the next char is NOT one of the forbidden chars above or

         // whitespace, then this is the start of a plain scalar.

         return Constant.NULL_BL_T_LINEBR.hasNo(ch, "-?:,[]{}#&*!|>\'\"%@`")

                 || (Constant.NULL_BL_T_LINEBR.hasNo(reader.peek(1)) && (ch == '-' || (this.flowLevel == 0 && "?:"

                         .indexOf(ch) != -1)));

     }

     // Scanners.

     /**

      * <pre>

      * We ignore spaces, line breaks and comments.

      * If we find a line break in the block context, we set the flag

      * `allow_simple_key` on.

      * The byte order mark is stripped if it's the first character in the

      * stream. We do not yet support BOM inside the stream as the

      * specification requires. Any such mark will be considered as a part

      * of the document.

      * TODO: We need to make tab handling rules more sane. A good rule is

      *   Tabs cannot precede tokens

      *   BLOCK-SEQUENCE-START, BLOCK-MAPPING-START, BLOCK-END,

      *   KEY(block), VALUE(block), BLOCK-ENTRY

      * So the checking code is

      *   if <TAB>:

      *       self.allow_simple_keys = False

      * We also need to add the check for `allow_simple_keys == True` to

      * `unwind_indent` before issuing BLOCK-END.

      * Scanners for block, flow, and plain scalars need to be modified.

      * </pre>

      */

     private void scanToNextToken() {

         // If there is a byte order mark (BOM) at the beginning of the stream,

         // forward past it.

         if (reader.getIndex() == 0 && reader.peek() == '\uFEFF') {

             reader.forward();

         }

         boolean found = false;

         while (!found) {

             int ff = 0;

             // Peek ahead until we find the first non-space character, then

             // move forward directly to that character.

             while (reader.peek(ff) == ' ') {

                 ff++;

             }

             if (ff > 0) {

                 reader.forward(ff);

             }

             // If the character we have skipped forward to is a comment (#),

             // then peek ahead until we find the next end of line. YAML

             // comments are from a # to the next new-line. We then forward

             // past the comment.

             if (reader.peek() == '#') {

                 ff = 0;

                 while (Constant.NULL_OR_LINEBR.hasNo(reader.peek(ff))) {

                     ff++;

                 }

                 if (ff > 0) {

                     reader.forward(ff);

                 }

             }

             // If we scanned a line break, then (depending on flow level),

             // simple keys may be allowed.

             if (scanLineBreak().length() != 0) {// found a line-break

                 if (this.flowLevel == 0) {

                     // Simple keys are allowed at flow-level 0 after a line

                     // break

                     this.allowSimpleKey = true;

                 }

             } else {

                 found = true;

             }

         }

     }

     @SuppressWarnings({ "unchecked", "rawtypes" })

     private Token scanDirective() {

         // See the specification for details.

         Mark startMark = reader.getMark();

         Mark endMark;

         reader.forward();

         String name = scanDirectiveName(startMark);

         List<?> value = null;

         if ("YAML".equals(name)) {

             value = scanYamlDirectiveValue(startMark);

             endMark = reader.getMark();

         } else if ("TAG".equals(name)) {

             value = scanTagDirectiveValue(startMark);

             endMark = reader.getMark();

         } else {

             endMark = reader.getMark();

             int ff = 0;

             while (Constant.NULL_OR_LINEBR.hasNo(reader.peek(ff))) {

                 ff++;

             }

             if (ff > 0) {

                 reader.forward(ff);

             }

         }

         scanDirectiveIgnoredLine(startMark);

         return new DirectiveToken(name, value, startMark, endMark);

     }

     /**

      * Scan a directive name. Directive names are a series of non-space

      * characters.

      * 

      * @see http://www.yaml.org/spec/1.1/#id895217

      */

     private String scanDirectiveName(Mark startMark) {

         // See the specification for details.

         int length = 0;

         // A Directive-name is a sequence of alphanumeric characters

         // (a-z,A-Z,0-9). We scan until we find something that isn't.

         // FIXME this disagrees with the specification.

         char ch = reader.peek(length);

         while (Constant.ALPHA.has(ch)) {

             length++;

             ch = reader.peek(length);

         }

         // If the name would be empty, an error occurs.

         if (length == 0) {

             throw new ScannerException("while scanning a directive", startMark,

                     "expected alphabetic or numeric character, but found " + ch + "(" + ((int) ch)

                             + ")", reader.getMark());

         }

         String value = reader.prefixForward(length);

         ch = reader.peek();

         if (Constant.NULL_BL_LINEBR.hasNo(ch)) {

             throw new ScannerException("while scanning a directive", startMark,

                     "expected alphabetic or numeric character, but found " + ch + "(" + ((int) ch)

                             + ")", reader.getMark());

         }

         return value;

     }

     private List<Integer> scanYamlDirectiveValue(Mark startMark) {

         // See the specification for details.

         while (reader.peek() == ' ') {

             reader.forward();

         }

         Integer major = scanYamlDirectiveNumber(startMark);

         if (reader.peek() != '.') {

             throw new ScannerException("while scanning a directive", startMark,

                     "expected a digit or '.', but found " + reader.peek() + "("

                             + ((int) reader.peek()) + ")", reader.getMark());

         }

         reader.forward();

         Integer minor = scanYamlDirectiveNumber(startMark);

         if (Constant.NULL_BL_LINEBR.hasNo(reader.peek())) {

             throw new ScannerException("while scanning a directive", startMark,

                     "expected a digit or ' ', but found " + reader.peek() + "("

                             + ((int) reader.peek()) + ")", reader.getMark());

         }

         List<Integer> result = new ArrayList<Integer>(2);

         result.add(major);

         result.add(minor);

         return result;

     }

     /**

      * Read a %YAML directive number: this is either the major or the minor

      * part. Stop reading at a non-digit character (usually either '.' or '\n').

      * 

      * @see http://www.yaml.org/spec/1.1/#id895631

      * @see http://www.yaml.org/spec/1.1/#ns-dec-digit

      */

     private Integer scanYamlDirectiveNumber(Mark startMark) {

         // See the specification for details.

         char ch = reader.peek();

         if (!Character.isDigit(ch)) {

             throw new ScannerException("while scanning a directive", startMark,

                     "expected a digit, but found " + ch + "(" + ((int) ch) + ")", reader.getMark());