Java 类com.facebook.presto.metadata.MetadataManager 实例源码

@Inject
public ConnectorManager(MetadataManager metadataManager,
        AccessControlManager accessControlManager,
        SplitManager splitManager,
        PageSourceManager pageSourceManager,
        IndexManager indexManager,
        PageSinkManager pageSinkManager,
        HandleResolver handleResolver,
        NodeManager nodeManager,
        TransactionManager transactionManager)
{
    this.metadataManager = metadataManager;
    this.accessControlManager = accessControlManager;
    this.splitManager = splitManager;
    this.pageSourceManager = pageSourceManager;
    this.indexManager = indexManager;
    this.pageSinkManager = pageSinkManager;
    this.handleResolver = handleResolver;
    this.nodeManager = nodeManager;
    this.transactionManager = transactionManager;
}

@Inject
public DataDefinitionExecutionFactory(
        LocationFactory locationFactory,
        TransactionManager transactionManager,
        MetadataManager metadata,
        AccessControl accessControl,
        @ForQueryExecution ExecutorService executor,
        Map<Class<? extends Statement>, DataDefinitionTask<?>> tasks)
{
    this.locationFactory = requireNonNull(locationFactory, "locationFactory is null");
    this.transactionManager = requireNonNull(transactionManager, "transactionManager is null");
    this.metadata = requireNonNull(metadata, "metadata is null");
    this.accessControl = requireNonNull(accessControl, "accessControl is null");
    this.executor = requireNonNull(executor, "executor is null");
    this.tasks = requireNonNull(tasks, "tasks is null");
}

@Test
public void testUnnestNonNumericDoubles()
        throws Exception
{
    MetadataManager metadata = createTestMetadataManager();
    Type arrayType = metadata.getType(parseTypeSignature("array<double>"));
    Type mapType = metadata.getType(parseTypeSignature("map<bigint,double>"));

    List<Page> input = rowPagesBuilder(BIGINT, arrayType, mapType)
            .row(1, arrayBlockOf(DOUBLE, NEGATIVE_INFINITY, POSITIVE_INFINITY, NaN),
                    mapBlockOf(BIGINT, DOUBLE, ImmutableMap.of(1, NEGATIVE_INFINITY, 2, POSITIVE_INFINITY, 3, NaN)))
            .build();

    OperatorFactory operatorFactory = new UnnestOperator.UnnestOperatorFactory(
            0, new PlanNodeId("test"), ImmutableList.of(0), ImmutableList.<Type>of(BIGINT), ImmutableList.of(1, 2), ImmutableList.of(arrayType, mapType), false);
    Operator operator = operatorFactory.createOperator(driverContext);

    MaterializedResult expected = resultBuilder(driverContext.getSession(), BIGINT, DOUBLE, BIGINT, DOUBLE)
            .row(1, NEGATIVE_INFINITY, 1, NEGATIVE_INFINITY)
            .row(1, POSITIVE_INFINITY, 2, POSITIVE_INFINITY)
            .row(1, NaN, 3, NaN)
            .build();

    assertOperatorEquals(operator, input, expected);
}

@Test
public void testNoCaching()
        throws Throwable
{
    MetadataManager metadata = MetadataManager.createTestMetadataManager();
    ExpressionCompiler compiler = new ExpressionCompiler(metadata);
    ImmutableList.Builder<RowExpression> projectionsBuilder = ImmutableList.builder();
    ArrayType arrayType = new ArrayType(VARCHAR);
    Signature signature = new Signature("concat", FunctionKind.SCALAR, arrayType.getTypeSignature(), arrayType.getTypeSignature(), arrayType.getTypeSignature());
    projectionsBuilder.add(new CallExpression(signature, arrayType, ImmutableList.of(new InputReferenceExpression(0, arrayType), new InputReferenceExpression(1, arrayType))));

    ImmutableList<RowExpression> projections = projectionsBuilder.build();
    PageProcessor pageProcessor = compiler.compilePageProcessor(new ConstantExpression(true, BooleanType.BOOLEAN), projections);
    PageProcessor pageProcessor2 = compiler.compilePageProcessor(new ConstantExpression(true, BooleanType.BOOLEAN), projections);
    assertTrue(pageProcessor != pageProcessor2);
}

@Setup
public void setup()
{
    MetadataManager metadata = MetadataManager.createTestMetadataManager();
    metadata.addFunctions(new FunctionListBuilder(metadata.getTypeManager()).scalar(BenchmarkArrayDistinct.class).getFunctions());
    ExpressionCompiler compiler = new ExpressionCompiler(metadata);
    ImmutableList.Builder<RowExpression> projectionsBuilder = ImmutableList.builder();
    Block[] blocks = new Block[TYPES.size()];
    for (int i = 0; i < TYPES.size(); i++) {
        Type elementType = TYPES.get(i);
        ArrayType arrayType = new ArrayType(elementType);
        Signature signature = new Signature(name, FunctionKind.SCALAR, arrayType.getTypeSignature(), arrayType.getTypeSignature());
        projectionsBuilder.add(new CallExpression(signature, arrayType, ImmutableList.of(new InputReferenceExpression(i, arrayType))));
        blocks[i] = createChannel(POSITIONS, ARRAY_SIZE, arrayType);
    }

    ImmutableList<RowExpression> projections = projectionsBuilder.build();
    pageProcessor = compiler.compilePageProcessor(new ConstantExpression(true, BooleanType.BOOLEAN), projections);
    pageBuilder = new PageBuilder(projections.stream().map(RowExpression::getType).collect(Collectors.toList()));
    page = new Page(blocks);
}

public static LocalExecutionPlanner createTestingPlanner()
{
    MetadataManager metadata = MetadataManager.createTestMetadataManager();

    PageSourceManager pageSourceManager = new PageSourceManager();
    pageSourceManager.addConnectorPageSourceProvider("test", new TestingPageSourceProvider());
    return new LocalExecutionPlanner(
            metadata,
            new SqlParser(),
            pageSourceManager,
            new IndexManager(),
            new PageSinkManager(),
            new MockExchangeClientSupplier(),
            new ExpressionCompiler(metadata),
            new IndexJoinLookupStats(),
            new CompilerConfig(),
            new TaskManagerConfig());
}

@Inject
public ExpressionCompiler() {
    TransactionManager transactionManager = TransactionManager.createTestTransactionManager();
    Metadata metadata = MetadataManager.createTestMetadataManager();

    this.serde = metadata.getBlockEncodingSerde();
    this.metadata = metadata;
    this.featuresConfig = new FeaturesConfig();
    this.typeManager = metadata.getTypeManager();
    this.session = Session.builder(new SessionPropertyManager())
            .setIdentity(new Identity("user", Optional.empty()))
            .setTimeZoneKey(TimeZoneKey.UTC_KEY)
            .setLocale(Locale.ENGLISH)
            .setQueryId(QueryId.valueOf("row_expression_compiler"))
            .setTransactionId(transactionManager.beginTransaction(IsolationLevel.REPEATABLE_READ, true, true))
            .build();

    this.expressionOptimizer = new ExpressionOptimizer(metadata.getFunctionRegistry(), metadata.getTypeManager(), session);

}

@Test
public void testUnnest()
        throws Exception
{
    MetadataManager metadata = createTestMetadataManager();
    Type arrayType = metadata.getType(parseTypeSignature("array<bigint>"));
    Type mapType = metadata.getType(parseTypeSignature("map<bigint,bigint>"));

    List<Page> input = rowPagesBuilder(BIGINT, arrayType, mapType)
            .row(1, arrayBlockOf(BIGINT, 2, 3), mapBlockOf(BIGINT, BIGINT, ImmutableMap.of(4, 5)))
            .row(2, arrayBlockOf(BIGINT, 99), null)
            .row(3, null, null)
            .pageBreak()
            .row(6, arrayBlockOf(BIGINT, 7, 8), mapBlockOf(BIGINT, BIGINT, ImmutableMap.of(9, 10, 11, 12)))
            .build();

    OperatorFactory operatorFactory = new UnnestOperator.UnnestOperatorFactory(
            0, new PlanNodeId("test"), ImmutableList.of(0), ImmutableList.<Type>of(BIGINT), ImmutableList.of(1, 2), ImmutableList.of(arrayType, mapType), false);
    Operator operator = operatorFactory.createOperator(driverContext);

    MaterializedResult expected = resultBuilder(driverContext.getSession(), BIGINT, BIGINT, BIGINT, BIGINT)
            .row(1, 2, 4, 5)
            .row(1, 3, null, null)
            .row(2, 99, null, null)
            .row(6, 7, 9, 10)
            .row(6, 8, 11, 12)
            .build();

    assertOperatorEquals(operator, input, expected);
}

@Test
public void testUnnestWithArray()
        throws Exception
{
    MetadataManager metadata = createTestMetadataManager();
    Type arrayType = metadata.getType(parseTypeSignature("array<array<bigint>>"));
    Type mapType = metadata.getType(parseTypeSignature("map<array<bigint>,array<bigint>>"));

    List<Page> input = rowPagesBuilder(BIGINT, arrayType, mapType)
            .row(
                    1,
                    arrayBlockOf(new ArrayType(BIGINT), ImmutableList.of(2, 4), ImmutableList.of(3, 6)),
                    mapBlockOf(new ArrayType(BIGINT), new ArrayType(BIGINT), ImmutableMap.of(ImmutableList.of(4, 8), ImmutableList.of(5, 10))))
            .row(2, arrayBlockOf(new ArrayType(BIGINT), ImmutableList.of(99, 198)), null)
            .row(3, null, null)
            .pageBreak()
            .row(
                    6,
                    arrayBlockOf(new ArrayType(BIGINT), ImmutableList.of(7, 14), ImmutableList.of(8, 16)),
                    mapBlockOf(new ArrayType(BIGINT), new ArrayType(BIGINT), ImmutableMap.of(ImmutableList.of(9, 18), ImmutableList.of(10, 20), ImmutableList.of(11, 22), ImmutableList.of(12, 24))))
            .build();

    OperatorFactory operatorFactory = new UnnestOperator.UnnestOperatorFactory(
            0, new PlanNodeId("test"), ImmutableList.of(0), ImmutableList.<Type>of(BIGINT), ImmutableList.of(1, 2), ImmutableList.of(arrayType, mapType), false);
    Operator operator = operatorFactory.createOperator(driverContext);

    MaterializedResult expected = resultBuilder(driverContext.getSession(), BIGINT, new ArrayType(BIGINT), new ArrayType(BIGINT), new ArrayType(BIGINT))
            .row(1, ImmutableList.of(2L, 4L), ImmutableList.of(4L, 8L), ImmutableList.of(5L, 10L))
            .row(1, ImmutableList.of(3L, 6L), null, null)
            .row(2, ImmutableList.of(99L, 198L), null, null)
            .row(6, ImmutableList.of(7L, 14L), ImmutableList.of(9L, 18L), ImmutableList.of(10L, 20L))
            .row(6, ImmutableList.of(8L, 16L), ImmutableList.of(11L, 22L), ImmutableList.of(12L, 24L))
            .build();

    assertOperatorEquals(operator, input, expected);
}

@Test
public void testUnnestWithOrdinality()
        throws Exception
{
    MetadataManager metadata = createTestMetadataManager();
    Type arrayType = metadata.getType(parseTypeSignature("array<bigint>"));
    Type mapType = metadata.getType(parseTypeSignature("map<bigint,bigint>"));

    List<Page> input = rowPagesBuilder(BIGINT, arrayType, mapType)
            .row(1, arrayBlockOf(BIGINT, 2, 3), mapBlockOf(BIGINT, BIGINT, ImmutableMap.of(4, 5)))
            .row(2, arrayBlockOf(BIGINT, 99), null)
            .row(3, null, null)
            .pageBreak()
            .row(6, arrayBlockOf(BIGINT, 7, 8), mapBlockOf(BIGINT, BIGINT, ImmutableMap.of(9, 10, 11, 12)))
            .build();

    OperatorFactory operatorFactory = new UnnestOperator.UnnestOperatorFactory(
            0, new PlanNodeId("test"), ImmutableList.of(0), ImmutableList.<Type>of(BIGINT), ImmutableList.of(1, 2), ImmutableList.of(arrayType, mapType), true);
    Operator operator = operatorFactory.createOperator(driverContext);

    MaterializedResult expected = resultBuilder(driverContext.getSession(), BIGINT, BIGINT, BIGINT, BIGINT, BIGINT)
            .row(1, 2, 4, 5, 1)
            .row(1, 3, null, null, 2)
            .row(2, 99, null, null, 1)
            .row(6, 7, 9, 10, 1)
            .row(6, 8, 11, 12, 2)
            .build();

    assertOperatorEquals(operator, input, expected);
}

@Test
public void testAggregation()
        throws Exception
{
    MetadataManager metadata = MetadataManager.createTestMetadataManager();
    InternalAggregationFunction countVarcharColumn = metadata.getFunctionRegistry().getAggregateFunctionImplementation(new Signature("count", AGGREGATE, StandardTypes.BIGINT, StandardTypes.VARCHAR));
    InternalAggregationFunction maxVarcharColumn = metadata.getFunctionRegistry().getAggregateFunctionImplementation(new Signature("max", AGGREGATE, StandardTypes.VARCHAR, StandardTypes.VARCHAR));
    List<Page> input = rowPagesBuilder(VARCHAR, BIGINT, VARCHAR, BIGINT, DOUBLE, VARCHAR)
            .addSequencePage(100, 0, 0, 300, 500, 500, 500)
            .build();

    OperatorFactory operatorFactory = new AggregationOperatorFactory(
            0,
            new PlanNodeId("test"),
            Step.SINGLE,
            ImmutableList.of(COUNT.bind(ImmutableList.of(0), Optional.empty(), Optional.empty(), 1.0),
                    LONG_SUM.bind(ImmutableList.of(1), Optional.empty(), Optional.empty(), 1.0),
                    LONG_AVERAGE.bind(ImmutableList.of(1), Optional.empty(), Optional.empty(), 1.0),
                    maxVarcharColumn.bind(ImmutableList.of(2), Optional.empty(), Optional.empty(), 1.0),
                    countVarcharColumn.bind(ImmutableList.of(0), Optional.empty(), Optional.empty(), 1.0),
                    LONG_SUM.bind(ImmutableList.of(3), Optional.empty(), Optional.empty(), 1.0),
                    DOUBLE_SUM.bind(ImmutableList.of(4), Optional.empty(), Optional.empty(), 1.0),
                    maxVarcharColumn.bind(ImmutableList.of(5), Optional.empty(), Optional.empty(), 1.0)));
    Operator operator = operatorFactory.createOperator(driverContext);

    MaterializedResult expected = resultBuilder(driverContext.getSession(), BIGINT, BIGINT, DOUBLE, VARCHAR, BIGINT, BIGINT, DOUBLE, VARCHAR)
            .row(100, 4950, 49.5, "399", 100, 54950, 54950.0, "599")
            .build();

    assertOperatorEquals(operator, input, expected);
}

@Setup
public void setup()
{
    inputPage = createInputPage();

    handCodedProcessor = new Tpch1FilterAndProject();

    compiledProcessor = new ExpressionCompiler(MetadataManager.createTestMetadataManager()).compilePageProcessor(FILTER, ImmutableList.of(PROJECT));
}

public SqlEngine(StructuredStore store, ExecutorService executor) {
    this.store = store;
    this.executor = executor;
    MetadataManager metadataManager = new MetadataManager();

    SplitManager splitManager = new SplitManager(Sets.<ConnectorSplitManager> newHashSet());

    this.dataStreamManager = new DataStreamManager();
    HandleResolver handleResolver = new HandleResolver();
    Map<String, ConnectorFactory> connectorFactories = Maps.newHashMap();
    Map<String, Connector> globalConnectors = Maps.newHashMap();

    RecordSinkManager recordSinkManager = new RecordSinkManager();
    Map<String, ConnectorOutputHandleResolver> handleIdResolvers = Maps.newHashMap();
    OutputTableHandleResolver outputTableHandleResolver = new OutputTableHandleResolver(handleIdResolvers);

    this.connectorManager = new ConnectorManager(metadataManager, splitManager, dataStreamManager,
            recordSinkManager, handleResolver, outputTableHandleResolver, connectorFactories, globalConnectors);

    // NodeManager nodeManager = new InMemoryNodeManager();
    PlanOptimizersFactory planOptimizersFactory = new PlanOptimizersFactory(metadataManager, splitManager);
    List<PlanOptimizer> planOptimizers = planOptimizersFactory.get();

    this.metadataManager = metadataManager;
    this.planOptimizers = planOptimizers;
    this.periodicImportManager = new StubPeriodicImportManager();
    this.storageManager = new StubStorageManager();

    NodeManager nodeManager = new InMemoryNodeManager();
    CloudataConnectorFactory cloudataConnectorFactory = new CloudataConnectorFactory(nodeManager,
            Maps.<String, String> newHashMap(), store);

    connectorManager.addConnectorFactory(cloudataConnectorFactory);

    connectorManager.createConnection(catalogName, CloudataConnectorFactory.PROVIDER_ID,
            Maps.<String, String> newHashMap());

    this.cloudataConnector = cloudataConnectorFactory.get(catalogName);
}

private Plan parse(String sql) {
    InMemoryNodeManager nodeManager = new InMemoryNodeManager();

    MetadataManager metadata = buildMetadata();
    StorageManager storageManager = new MockStorageManager();
    PeriodicImportManager periodicImportManager = new StubPeriodicImportManager();

    SplitManager splitManager = buildSplitManager(nodeManager);
    List<PlanOptimizer> planOptimizers = buildPlanOptimizers(metadata, splitManager);

    Statement statement = SqlParser.createStatement(sql);

    // System.out.println("Statement: " + statement);

    Session session = buildSession();
    QueryExplainer queryExplainer = new QueryExplainer(session, planOptimizers, metadata, periodicImportManager,
            storageManager);
    // analyze query
    Analyzer analyzer = new Analyzer(session, metadata, Optional.of(queryExplainer));

    Analysis analysis = analyzer.analyze(statement);

    // System.out.println("analysis: " + analysis);

    PlanNodeIdAllocator idAllocator = new PlanNodeIdAllocator();
    // plan query
    LogicalPlanner logicalPlanner = new LogicalPlanner(session, planOptimizers, idAllocator, metadata,
            periodicImportManager, storageManager);
    Plan plan = logicalPlanner.plan(analysis);
    return plan;
}

private MetadataManager buildMetadata() {
    // Metadata metadata = new TestMetadata();
    MetadataManager metadata = new MetadataManager();

    StructuredStore store = null;
    metadata.addConnectorMetadata(connectorId, catalog, new CloudataConnectorMetadata(connectorId, store));

    return metadata;
}

private List<PlanOptimizer> buildPlanOptimizers(MetadataManager metadata, SplitManager splitManager) {
    PlanOptimizersFactory planOptimizersFactory = new PlanOptimizersFactory(metadata, splitManager);
    List<PlanOptimizer> planOptimizers = planOptimizersFactory.get();
    return planOptimizers;
}