CodeExplorer
You are a developer and looking for Shopware projects?
Apply Now!
Token example
use PHPUnit\Framework\TestCase;
use Symfony\Component\CssSelector\Exception\SyntaxErrorException;
use Symfony\Component\CssSelector\Parser\Token;
use Symfony\Component\CssSelector\Parser\TokenStream;
class TokenStreamTest extends TestCase
{
public function testGetNext()
{
$stream = new TokenStream();
$stream->push($t1 = new Token(Token::TYPE_IDENTIFIER, 'h1', 0));
$stream->push($t2 = new Token(Token::TYPE_DELIMITER, '.', 2));
$stream->push($t3 = new Token(Token::TYPE_IDENTIFIER, 'title', 3));
$this->assertSame($t1, $stream->getNext());
$this->assertSame($t2, $stream->getNext());
$this->assertSame($t3, $stream->getNext());
}
public function testGetPeek()
{
$stream = new TokenStream();
use Symfony\Component\CssSelector\Exception\SyntaxErrorException;
use Symfony\Component\CssSelector\Parser\Token;
use Symfony\Component\CssSelector\Parser\TokenStream;
class TokenStreamTest extends TestCase
{
public function testGetNext()
{
$stream = new TokenStream();
$stream->push($t1 = new Token(Token::TYPE_IDENTIFIER, 'h1', 0));
$stream->push($t2 = new Token(Token::TYPE_DELIMITER, '.', 2));
$stream->push($t3 = new Token(Token::TYPE_IDENTIFIER, 'title', 3));
$this->assertSame($t1, $stream->getNext());
$this->assertSame($t2, $stream->getNext());
$this->assertSame($t3, $stream->getNext());
}
public function testGetPeek()
{
$stream = new TokenStream();
public function handle(Reader $reader, TokenStream $stream): bool
{
$match = $reader->findPattern($this->patterns->getHashPattern());
if (!$match) {
return false;
}
$value = $this->escaping->escapeUnicode($match[1]);
$stream->push(new Token(Token::TYPE_HASH, $value, $reader->getPosition()));
$reader->moveForward(\strlen($match[0]));
return true;
}
}
use Symfony\Component\CssSelector\Parser\Handler\IdentifierHandler;
use Symfony\Component\CssSelector\Parser\Token;
use Symfony\Component\CssSelector\Parser\Tokenizer\TokenizerEscaping;
use Symfony\Component\CssSelector\Parser\Tokenizer\TokenizerPatterns;
class IdentifierHandlerTest extends AbstractHandlerTestCase
{
public static function getHandleValueTestData()
{
return [
['foo', new Token(Token::TYPE_IDENTIFIER, 'foo', 0), ''],
['foo|bar', new Token(Token::TYPE_IDENTIFIER, 'foo', 0), '|bar'],
['foo.class', new Token(Token::TYPE_IDENTIFIER, 'foo', 0), '.class'],
['foo[attr]', new Token(Token::TYPE_IDENTIFIER, 'foo', 0), '[attr]'],
['foo bar', new Token(Token::TYPE_IDENTIFIER, 'foo', 0), ' bar'],
];
}
public static function getDontHandleValueTestData()
{
return [
['>'],
[
use Symfony\Component\CssSelector\Parser\Token;
use Symfony\Component\CssSelector\Parser\Tokenizer\TokenizerEscaping;
use Symfony\Component\CssSelector\Parser\Tokenizer\TokenizerPatterns;
class IdentifierHandlerTest extends AbstractHandlerTestCase
{
public static function getHandleValueTestData()
{
return [
['foo', new Token(Token::TYPE_IDENTIFIER, 'foo', 0), ''],
['foo|bar', new Token(Token::TYPE_IDENTIFIER, 'foo', 0), '|bar'],
['foo.class', new Token(Token::TYPE_IDENTIFIER, 'foo', 0), '.class'],
['foo[attr]', new Token(Token::TYPE_IDENTIFIER, 'foo', 0), '[attr]'],
['foo bar', new Token(Token::TYPE_IDENTIFIER, 'foo', 0), ' bar'],
];
}
public static function getDontHandleValueTestData()
{
return [
['>'],
[
$this->openBrackets[$nextVal]--;
}
}
$this->nextToken = null;
}
//If comments handling is enabled, get the comments associated with the //current token $comments = $this->comments ? $this->commentsForCurrentToken() : null;
//Replace the current token with a regexp token $token = new Token(Token::TYPE_REGULAR_EXPRESSION, $buffer);
$token->location->start = $startPosition;
$token->location->end = $this->getPosition(true);
$this->currentToken = $token;
if ($comments) {
//Attach the comments to the new current token $this->commentsForCurrentToken($comments);
}
return $this->currentToken;
}
}
}
$this->nextToken = null;
}
//If comments handling is enabled, get the comments associated with the //current token $comments = $this->comments ? $this->commentsForCurrentToken() : null;
//Replace the current token with a regexp token $token = new Token(Token::TYPE_REGULAR_EXPRESSION, $buffer);
$token->location->start = $startPosition;
$token->location->end = $this->getPosition(true);
$this->currentToken = $token;
if ($comments) {
//Attach the comments to the new current token $this->commentsForCurrentToken($comments);
}
return $this->currentToken;
}
final class EmbedTokenParser extends IncludeTokenParser
{
public function parse(Token $token): Node
{
$stream = $this->parser->getStream();
$parent = $this->parser->getExpressionParser()->parseExpression();
list($variables, $only, $ignoreMissing) = $this->parseArguments();
$parentToken = $fakeParentToken = new Token(/* Token::STRING_TYPE */ 7, '__parent__', $token->getLine());
if ($parent instanceof ConstantExpression) {
$parentToken = new Token(/* Token::STRING_TYPE */ 7, $parent->getAttribute('value'), $token->getLine());
} elseif ($parent instanceof NameExpression) {
$parentToken = new Token(/* Token::NAME_TYPE */ 5, $parent->getAttribute('name'), $token->getLine());
}
// inject a fake parent to make the parent() function work $stream->injectTokens([
new Token(/* Token::BLOCK_START_TYPE */ 1, '', $token->getLine()),
new Token(/* Token::NAME_TYPE */ 5, 'extends', $token->getLine()),
$parentToken,
++$cursor;
continue;
}
if (preg_match('/ (?(DEFINE)(?P<LNUM>[0-9]+(_[0-9]+)*)) (?:\.(?&LNUM)|(?&LNUM)(?:\.(?!\.)(?&LNUM)?)?)(?:[eE][+-]?(?&LNUM))?/Ax',
$expression, $match, 0, $cursor)
) {
// numbers $tokens[] = new Token(Token::NUMBER_TYPE, 0 + str_replace('_', '', $match[0]), $cursor + 1);
$cursor += \strlen($match[0]);
} elseif (str_contains('([{', $expression[$cursor])) {
// opening bracket $brackets[] = [$expression[$cursor], $cursor];
$tokens[] = new Token(Token::PUNCTUATION_TYPE, $expression[$cursor], $cursor + 1);
++$cursor;
} elseif (str_contains(')]}', $expression[$cursor])) {
// closing bracket if (!$brackets) {
throw new SyntaxError(sprintf('Unexpected "%s".', $expression[$cursor]), $cursor, $expression);
}
continue;
}
if (preg_match('/ (?(DEFINE)(?P<LNUM>[0-9]+(_[0-9]+)*)) (?:\.(?&LNUM)|(?&LNUM)(?:\.(?!\.)(?&LNUM)?)?)(?:[eE][+-]?(?&LNUM))?/Ax',
$expression, $match, 0, $cursor)
) {
// numbers $tokens[] = new Token(Token::NUMBER_TYPE, 0 + str_replace('_', '', $match[0]), $cursor + 1);
$cursor += \strlen($match[0]);
} elseif (str_contains('([{', $expression[$cursor])) {
// opening bracket $brackets[] = [$expression[$cursor], $cursor];
$tokens[] = new Token(Token::PUNCTUATION_TYPE, $expression[$cursor], $cursor + 1);
++$cursor;
} elseif (str_contains(')]}', $expression[$cursor])) {
// closing bracket if (!$brackets) {
throw new SyntaxError(sprintf('Unexpected "%s".', $expression[$cursor]), $cursor, $expression);
}
use Symfony\Component\CssSelector\Parser\Handler\StringHandler;
use Symfony\Component\CssSelector\Parser\Token;
use Symfony\Component\CssSelector\Parser\Tokenizer\TokenizerEscaping;
use Symfony\Component\CssSelector\Parser\Tokenizer\TokenizerPatterns;
class StringHandlerTest extends AbstractHandlerTestCase
{
public static function getHandleValueTestData()
{
return [
['"hello"', new Token(Token::TYPE_STRING, 'hello', 1), ''],
['"1"', new Token(Token::TYPE_STRING, '1', 1), ''],
['" "', new Token(Token::TYPE_STRING, ' ', 1), ''],
['""', new Token(Token::TYPE_STRING, '', 1), ''],
["'hello'", new Token(Token::TYPE_STRING, 'hello', 1), ''],
["'foo'bar", new Token(Token::TYPE_STRING, 'foo', 1), 'bar'],
];
}
public static function getDontHandleValueTestData()
{
use Symfony\Component\CssSelector\Parser\Token;
use Symfony\Component\CssSelector\Parser\Tokenizer\TokenizerEscaping;
use Symfony\Component\CssSelector\Parser\Tokenizer\TokenizerPatterns;
class StringHandlerTest extends AbstractHandlerTestCase
{
public static function getHandleValueTestData()
{
return [
['"hello"', new Token(Token::TYPE_STRING, 'hello', 1), ''],
['"1"', new Token(Token::TYPE_STRING, '1', 1), ''],
['" "', new Token(Token::TYPE_STRING, ' ', 1), ''],
['""', new Token(Token::TYPE_STRING, '', 1), ''],
["'hello'", new Token(Token::TYPE_STRING, 'hello', 1), ''],
["'foo'bar", new Token(Token::TYPE_STRING, 'foo', 1), 'bar'],
];
}
public static function getDontHandleValueTestData()
{
private Token $nullToken;
/** @var string */
private $accumulator = '';
/** @var bool */
private $hasToRecord = false;
public function __construct()
{
/** @var Token<int, string> $nullToken */
$nullToken = new Token('', self::S_EMPTY, 0);
$this->nullToken = $nullToken;
$this->current = $this->previous = $this->nullToken;
$this->lookahead = null;
}
public function reset(): void
{
$this->hasInvalidTokens = false;
parent::reset();
$this->current = $this->previous = $this->nullToken;
}
/** @var string */
private $accumulator = '';
/** @var bool */
private $hasToRecord = false;
public function __construct()
{
/** @var Token<int, string> $nullToken */
$nullToken = new Token('', self::S_EMPTY, 0);
$this->nullToken = $nullToken;
$this->current = $this->previous = $this->nullToken;
$this->lookahead = null;
}
public function reset(): void
{
$this->hasInvalidTokens = false;
parent::reset();
$this->current = $this->previous = $this->nullToken;
}
class WhitespaceHandler implements HandlerInterface
{
public function handle(Reader $reader, TokenStream $stream): bool
{
$match = $reader->findPattern('~^[ \t\r\n\f]+~');
if (false === $match) {
return false;
}
$stream->push(new Token(Token::TYPE_WHITESPACE, $match[0], $reader->getPosition()));
$reader->moveForward(\strlen($match[0]));
return true;
}
}
namespace Symfony\Component\CssSelector\Tests\Parser\Handler;
use Symfony\Component\CssSelector\Parser\Handler\WhitespaceHandler;
use Symfony\Component\CssSelector\Parser\Token;
class WhitespaceHandlerTest extends AbstractHandlerTestCase
{
public static function getHandleValueTestData()
{
return [
[' ', new Token(Token::TYPE_WHITESPACE, ' ', 0), ''],
["\n", new Token(Token::TYPE_WHITESPACE, "\n", 0), ''],
["\t", new Token(Token::TYPE_WHITESPACE, "\t", 0), ''],
[' foo', new Token(Token::TYPE_WHITESPACE, ' ', 0), 'foo'],
[' .foo', new Token(Token::TYPE_WHITESPACE, ' ', 0), '.foo'],
];
}
public static function getDontHandleValueTestData()
{
return [
[
use Symfony\Component\CssSelector\Parser\Handler\WhitespaceHandler;
use Symfony\Component\CssSelector\Parser\Token;
class WhitespaceHandlerTest extends AbstractHandlerTestCase
{
public static function getHandleValueTestData()
{
return [
[' ', new Token(Token::TYPE_WHITESPACE, ' ', 0), ''],
["\n", new Token(Token::TYPE_WHITESPACE, "\n", 0), ''],
["\t", new Token(Token::TYPE_WHITESPACE, "\t", 0), ''],
[' foo', new Token(Token::TYPE_WHITESPACE, ' ', 0), 'foo'],
[' .foo', new Token(Token::TYPE_WHITESPACE, ' ', 0), '.foo'],
];
}
public static function getDontHandleValueTestData()
{
return [
[
$this->lexExpression();
}
}
private function pushToken($type, $value = ''): void
{
// do not push empty text tokens if (/* Token::TEXT_TYPE */ 0 === $type && '' === $value) {
return;
}
$this->tokens[] = new Token($type, $value, $this->lineno);
}
private function moveCursor($text): void
{
$this->cursor += \strlen($text);
$this->lineno += substr_count($text, "\n");
}
private function getOperatorRegex(): string
{
$operators = array_merge(
[
}
}
private function pushToken($type, $value = ''): void
{
// do not push empty text tokens if (/* Token::TEXT_TYPE */ 0 === $type && '' === $value) {
return;
}
$this->tokens[] = new Token($type, $value, $this->lineno);
}
private function moveCursor($text): void
{
$this->cursor += \strlen($text);
$this->lineno += substr_count($text, "\n");
}
private function getOperatorRegex(): string
{
$operators = array_merge(
[
if ($matches === false) {
// Work around https://bugs.php.net/78122 $matches = [[$input, 0]];
}
foreach ($matches as $match) {
// Must remain before 'value' assignment since it can change content $firstMatch = $match[0];
$type = $this->getType($firstMatch);
$this->tokens[] = new Token(
$firstMatch,
$type,
$match[1]
);
}
}
/** * Gets the literal for a given token. * * @param T $token * * @return int|string */
// Work around https://bugs.php.net/78122 $matches = [[$input, 0]];
}
foreach ($matches as $match) {
// Must remain before 'value' assignment since it can change content $firstMatch = $match[0];
$type = $this->getType($firstMatch);
$this->tokens[] = new Token(
$firstMatch,
$type,
$match[1]
);
}
}
/** * Gets the literal for a given token. * * @param T $token * * @return int|string */
public function tokenize(Reader $reader): TokenStream
{
$stream = new TokenStream();
while (!$reader->isEOF()) {
foreach ($this->handlers as $handler) {
if ($handler->handle($reader, $stream)) {
continue 2;
}
}
$stream->push(new Token(Token::TYPE_DELIMITER, $reader->getSubstring(1), $reader->getPosition()));
$reader->moveForward(1);
}
return $stream
->push(new Token(Token::TYPE_FILE_END, null, $reader->getPosition()))
->freeze();
}
}
{
$stream = new TokenStream();
while (!$reader->isEOF()) {
foreach ($this->handlers as $handler) {
if ($handler->handle($reader, $stream)) {
continue 2;
}
}
$stream->push(new Token(Token::TYPE_DELIMITER, $reader->getSubstring(1), $reader->getPosition()));
$reader->moveForward(1);
}
return $stream
->push(new Token(Token::TYPE_FILE_END, null, $reader->getPosition()))
->freeze();
}
}
use Symfony\Component\CssSelector\Node\ElementNode;
use Symfony\Component\CssSelector\Node\FunctionNode;
use Symfony\Component\CssSelector\Parser\Token;
class FunctionNodeTest extends AbstractNodeTestCase
{
public static function getToStringConversionTestData()
{
return [
[new FunctionNode(new ElementNode(), 'function'), 'Function[Element[*]:function()]'],
[new FunctionNode(new ElementNode(), 'function', [
new Token(Token::TYPE_IDENTIFIER, 'value', 0),
]), "Function[Element[*]:function(['value'])]"],
[new FunctionNode(new ElementNode(), 'function', [
new Token(Token::TYPE_STRING, 'value1', 0),
new Token(Token::TYPE_NUMBER, 'value2', 0),
]), "Function[Element[*]:function(['value1', 'value2'])]"],
];
}
public static function getSpecificityValueTestData()
{
return [
[
use Symfony\Component\CssSelector\Node\FunctionNode;
use Symfony\Component\CssSelector\Parser\Token;
class FunctionNodeTest extends AbstractNodeTestCase
{
public static function getToStringConversionTestData()
{
return [
[new FunctionNode(new ElementNode(), 'function'), 'Function[Element[*]:function()]'],
[new FunctionNode(new ElementNode(), 'function', [
new Token(Token::TYPE_IDENTIFIER, 'value', 0),
]), "Function[Element[*]:function(['value'])]"],
[new FunctionNode(new ElementNode(), 'function', [
new Token(Token::TYPE_STRING, 'value1', 0),
new Token(Token::TYPE_NUMBER, 'value2', 0),
]), "Function[Element[*]:function(['value1', 'value2'])]"],
];
}
public static function getSpecificityValueTestData()
{
return [
[
':scope > foo bar > div', ['CombinedSelector[CombinedSelector[CombinedSelector[Pseudo[Element[*]:scope] > Element[foo]] <followed> Element[bar]] > Element[div]]']],
[':scope > #foo #bar', ['CombinedSelector[CombinedSelector[Pseudo[Element[*]:scope] > Hash[Element[*]#foo]] <followed> Hash[Element[*]#bar]]']],
[':scope', ['Pseudo[Element[*]:scope]']],
['foo bar, :scope > div', ['CombinedSelector[Element[foo] <followed> Element[bar]]', 'CombinedSelector[Pseudo[Element[*]:scope] > Element[div]]']],
['foo bar,:scope > div', ['CombinedSelector[Element[foo] <followed> Element[bar]]', 'CombinedSelector[Pseudo[Element[*]:scope] > Element[div]]']],
];
}
public static function getParserExceptionTestData()
{
return [
['attributes(href)/html/body/a', SyntaxErrorException::unexpectedToken('selector', new Token(Token::TYPE_DELIMITER, '(', 10))->getMessage()],
['attributes(href)', SyntaxErrorException::unexpectedToken('selector', new Token(Token::TYPE_DELIMITER, '(', 10))->getMessage()],
['html/body/a', SyntaxErrorException::unexpectedToken('selector', new Token(Token::TYPE_DELIMITER, '/', 4))->getMessage()],
[' ', SyntaxErrorException::unexpectedToken('selector', new Token(Token::TYPE_FILE_END, '', 1))->getMessage()],
['div, ', SyntaxErrorException::unexpectedToken('selector', new Token(Token::TYPE_FILE_END, '', 5))->getMessage()],
[' , div', SyntaxErrorException::unexpectedToken('selector', new Token(Token::TYPE_DELIMITER, ',', 1))->getMessage()],
['p, , div', SyntaxErrorException::unexpectedToken('selector', new Token(Token::TYPE_DELIMITER, ',', 3))->getMessage()],
['div > ', SyntaxErrorException::unexpectedToken('selector', new Token(Token::TYPE_FILE_END, '', 6))->getMessage()],
[' > div', SyntaxErrorException::unexpectedToken('selector', new Token(Token::TYPE_DELIMITER, '>', 2))->getMessage()],
['foo|#bar', SyntaxErrorException::unexpectedToken('identifier or "*"', new Token(Token::TYPE_HASH, 'bar', 4))->getMessage()],
['#.foo', SyntaxErrorException::unexpectedToken('selector', new Token(Token::TYPE_DELIMITER, '#', 0))->getMessage()],
['.#foo', SyntaxErrorException::unexpectedToken('identifier', new Token(Token::TYPE_HASH, 'foo', 1))->getMessage()],
[
[':scope > #foo #bar', ['CombinedSelector[CombinedSelector[Pseudo[Element[*]:scope] > Hash[Element[*]#foo]] <followed> Hash[Element[*]#bar]]']],
[':scope', ['Pseudo[Element[*]:scope]']],
['foo bar, :scope > div', ['CombinedSelector[Element[foo] <followed> Element[bar]]', 'CombinedSelector[Pseudo[Element[*]:scope] > Element[div]]']],
['foo bar,:scope > div', ['CombinedSelector[Element[foo] <followed> Element[bar]]', 'CombinedSelector[Pseudo[Element[*]:scope] > Element[div]]']],
];
}
public static function getParserExceptionTestData()
{
return [
['attributes(href)/html/body/a', SyntaxErrorException::unexpectedToken('selector', new Token(Token::TYPE_DELIMITER, '(', 10))->getMessage()],
['attributes(href)', SyntaxErrorException::unexpectedToken('selector', new Token(Token::TYPE_DELIMITER, '(', 10))->getMessage()],
['html/body/a', SyntaxErrorException::unexpectedToken('selector', new Token(Token::TYPE_DELIMITER, '/', 4))->getMessage()],
[' ', SyntaxErrorException::unexpectedToken('selector', new Token(Token::TYPE_FILE_END, '', 1))->getMessage()],
['div, ', SyntaxErrorException::unexpectedToken('selector', new Token(Token::TYPE_FILE_END, '', 5))->getMessage()],
[' , div', SyntaxErrorException::unexpectedToken('selector', new Token(Token::TYPE_DELIMITER, ',', 1))->getMessage()],
['p, , div', SyntaxErrorException::unexpectedToken('selector', new Token(Token::TYPE_DELIMITER, ',', 3))->getMessage()],
['div > ', SyntaxErrorException::unexpectedToken('selector', new Token(Token::TYPE_FILE_END, '', 6))->getMessage()],
[' > div', SyntaxErrorException::unexpectedToken('selector', new Token(Token::TYPE_DELIMITER, '>', 2))->getMessage()],
['foo|#bar', SyntaxErrorException::unexpectedToken('identifier or "*"', new Token(Token::TYPE_HASH, 'bar', 4))->getMessage()],
['#.foo', SyntaxErrorException::unexpectedToken('selector', new Token(Token::TYPE_DELIMITER, '#', 0))->getMessage()],
['.#foo', SyntaxErrorException::unexpectedToken('identifier', new Token(Token::TYPE_HASH, 'foo', 1))->getMessage()],
[