Tokenizer example

class Parser implements ParserInterface
{
    private Tokenizer $tokenizer;

    public function __construct(Tokenizer $tokenizer = null)
    {
        $this->tokenizer = $tokenizer ?? new Tokenizer();
    }

    public function parse(string $source): array
    {
        $reader = new Reader($source);
        $stream = $this->tokenizer->tokenize($reader);

        return $this->parseSelectorList($stream);
    }

    /** * Parses the arguments for ":nth-child()" and friends. * * @param Token[] $tokens * * @throws SyntaxErrorException */

class ProductSearchQueryBuilderTest extends TestCase
{
    public function testDecoration(): void
    {
        $builder = new ProductSearchQueryBuilder(
            $this->createMock(Connection::class),
            new EntityDefinitionQueryHelper(),
            $this->getDefinition(),
            $this->createMock(TokenFilter::class),
            new Tokenizer(2),
            $this->createMock(ElasticsearchHelper::class)
        );

        static::expectException(DecorationPatternException::class);
        $builder->getDecorated();
    }

    public function testBuildQueryAndSearch(): void
    {
        $connection = $this->createMock(Connection::class);
        $connection
            

    public function parse($input, array $options = array())
    {
        $this->errors = array();
        $options = array_merge($this->defaultOptions, $options);
        $events = new DOMTreeBuilder(false, $options);
        $scanner = new Scanner($input, !empty($options['encoding']) ? $options['encoding'] : 'UTF-8');
        $parser = new Tokenizer($scanner, $events, !empty($options['xmlNamespaces']) ? Tokenizer::CONFORMANT_XML : Tokenizer::CONFORMANT_HTML);

        $parser->parse();
        $this->errors = $events->getErrors();

        return $events->document();
    }

    /** * Parse an input stream where the stream is a fragment. * * Lower-level loading function. This requires an input stream instead * of a string, file, or resource. * * @param string $input The input data to parse in the form of a string. * @param array $options An array of options. * * @return \DOMDocumentFragment */

/** * @internal */
class TokenizerTest extends TestCase
{
    /** * @dataProvider cases */
    public function testInterpreter(string $term, array $expected): void
    {
        $tokens = (new Tokenizer(2))->tokenize($term);
        static::assertSame($expected, $tokens);
    }

    public static function cases(): array
    {
        return [
            [
                ' ',
                [],
            ],
            [
                

    public function testAnalyze(array $productData, array $configFields, array $expected): void
    {
        $product = new ProductEntity();
        $product->assign($productData);

        $tokenizer = new Tokenizer(3);
        $tokenFilter = $this->createMock(TokenFilter::class);
        $tokenFilter->method('filter')->willReturnCallback(fn (array $tokens) => $tokens);

        $analyzer = new ProductSearchKeywordAnalyzer($tokenizer, $tokenFilter);
        $analyzer = $analyzer->analyze($product, $this->context, $configFields);
        $analyzerResult = $analyzer->getKeys();

        sort($analyzerResult);
        sort($expected);

        static::assertEquals($expected, $analyzerResult);
    }