@Override public Map<String, ColumnHandle> getColumnHandles(ConnectorSession session, ConnectorTableHandle tableHandle) { ExampleTableHandle exampleTableHandle = checkType(tableHandle, ExampleTableHandle.class, "tableHandle"); checkArgument(exampleTableHandle.getConnectorId().equals(connectorId), "tableHandle is not for this connector"); ExampleTable table = exampleClient.getTable(exampleTableHandle.getSchemaName(), exampleTableHandle.getTableName()); if (table == null) { throw new TableNotFoundException(exampleTableHandle.toSchemaTableName()); } ImmutableMap.Builder<String, ColumnHandle> columnHandles = ImmutableMap.builder(); int index = 0; for (ColumnMetadata column : table.getColumnsMetadata()) { columnHandles.put(column.getName(), new ExampleColumnHandle(connectorId, column.getName(), column.getType(), index)); index++; } return columnHandles.build(); }
@SuppressWarnings("ValueOfIncrementOrDecrementUsed") private ConnectorTableMetadata getTableMetadata(SchemaTableName schemaTableName) { RedisTableDescription table = getDefinedTables().get(schemaTableName); if (table == null) { throw new TableNotFoundException(schemaTableName); } ImmutableList.Builder<ColumnMetadata> builder = ImmutableList.builder(); appendFields(builder, table.getKey()); appendFields(builder, table.getValue()); for (RedisInternalFieldDescription fieldDescription : internalFieldDescriptions) { builder.add(fieldDescription.getColumnMetadata(hideInternalColumns)); } return new ConnectorTableMetadata(schemaTableName, builder.build()); }
@Override public Map<SchemaTableName, List<ColumnMetadata>> listTableColumns(ConnectorSession session, SchemaTablePrefix prefix) { ImmutableMap.Builder<SchemaTableName, List<ColumnMetadata>> columns = ImmutableMap.builder(); for (SchemaTableName tableName : listTables(session, prefix.getSchemaName())) { try { JdbcTableHandle tableHandle = jdbcClient.getTableHandle(tableName); if (tableHandle == null) { continue; } columns.put(tableName, getTableMetadata(session, tableHandle).getColumns()); } catch (TableNotFoundException e) { // table disappeared during listing operation } } return columns.build(); }
private TableMetadata getTableMetadata(SchemaTableName schemaTableName) { String schemaName = schemaTableName.getSchemaName(); String tableName = schemaTableName.getTableName(); KeyspaceMetadata keyspaceMetadata = getCheckedKeyspaceMetadata(schemaName); TableMetadata tableMetadata = keyspaceMetadata.getTable(tableName); if (tableMetadata != null) { return tableMetadata; } for (TableMetadata table : keyspaceMetadata.getTables()) { if (table.getName().equalsIgnoreCase(tableName)) { return table; } } throw new TableNotFoundException(schemaTableName); }
@Override public CassandraTable getTable(SchemaTableName tableName) throws TableNotFoundException { accessCount.incrementAndGet(); if (throwException) { throw new IllegalStateException(); } if (tableName.getSchemaName().equals(TEST_SCHEMA) && tableName.getTableName().equals(TEST_TABLE)) { return new CassandraTable( new CassandraTableHandle(connectorId, TEST_SCHEMA, TEST_TABLE), ImmutableList.of( new CassandraColumnHandle(connectorId, TEST_COLUMN1, 0, CassandraType.VARCHAR, null, true, false, false, false), new CassandraColumnHandle(connectorId, TEST_COLUMN2, 0, CassandraType.INT, null, false, false, false, false))); } throw new TableNotFoundException(tableName); }
@Override public Map<String, ColumnHandle> getColumnHandles(ConnectorSession session, ConnectorTableHandle tableHandle) { SchemaTableName tableName = schemaTableName(tableHandle); Optional<Table> table = metastore.getTable(tableName.getSchemaName(), tableName.getTableName()); if (!table.isPresent()) { throw new TableNotFoundException(tableName); } ImmutableMap.Builder<String, ColumnHandle> columnHandles = ImmutableMap.builder(); for (HiveColumnHandle columnHandle : hiveColumnHandles(connectorId, table.get())) { if (!columnHandle.getName().equals(SAMPLE_WEIGHT_COLUMN_NAME)) { columnHandles.put(columnHandle.getName(), columnHandle); } } return columnHandles.build(); }
@Override public void addColumn(ConnectorSession session, ConnectorTableHandle tableHandle, ColumnMetadata column) { if (!allowAddColumn) { throw new PrestoException(PERMISSION_DENIED, "Adding Columns is disabled in this Hive catalog"); } HiveTableHandle handle = checkType(tableHandle, HiveTableHandle.class, "tableHandle"); Optional<Table> tableMetadata = metastore.getTable(handle.getSchemaName(), handle.getTableName()); if (!tableMetadata.isPresent()) { throw new TableNotFoundException(handle.getSchemaTableName()); } Table table = tableMetadata.get(); StorageDescriptor sd = table.getSd(); ImmutableList.Builder<FieldSchema> columns = ImmutableList.builder(); columns.addAll(sd.getCols()); columns.add(new FieldSchema(column.getName(), toHiveType(column.getType()).getHiveTypeName(), column.getComment())); sd.setCols(columns.build()); table.setSd(sd); metastore.alterTable(handle.getSchemaName(), handle.getTableName(), table); }
@Override public void renameTable(ConnectorSession session, ConnectorTableHandle tableHandle, SchemaTableName newTableName) { if (!allowRenameTable) { throw new PrestoException(PERMISSION_DENIED, "Renaming tables is disabled in this Hive catalog"); } HiveTableHandle handle = checkType(tableHandle, HiveTableHandle.class, "tableHandle"); SchemaTableName tableName = schemaTableName(tableHandle); Optional<Table> source = metastore.getTable(handle.getSchemaName(), handle.getTableName()); if (!source.isPresent()) { throw new TableNotFoundException(tableName); } Table table = source.get(); table.setDbName(newTableName.getSchemaName()); table.setTableName(newTableName.getTableName()); metastore.alterTable(handle.getSchemaName(), handle.getTableName(), table); }
@Override public void dropTable(ConnectorSession session, ConnectorTableHandle tableHandle) { HiveTableHandle handle = checkType(tableHandle, HiveTableHandle.class, "tableHandle"); SchemaTableName tableName = schemaTableName(tableHandle); if (!allowDropTable) { throw new PrestoException(PERMISSION_DENIED, "DROP TABLE is disabled in this Hive catalog"); } Optional<Table> target = metastore.getTable(handle.getSchemaName(), handle.getTableName()); if (!target.isPresent()) { throw new TableNotFoundException(tableName); } Table table = target.get(); if (!session.getUser().equals(table.getOwner())) { throw new PrestoException(PERMISSION_DENIED, format("Unable to drop table '%s': owner of the table is different from session user", table)); } metastore.dropTable(handle.getSchemaName(), handle.getTableName()); }
@Override public HiveInsertTableHandle beginInsert(ConnectorSession session, ConnectorTableHandle tableHandle) { verifyJvmTimeZone(); SchemaTableName tableName = schemaTableName(tableHandle); Optional<Table> table = metastore.getTable(tableName.getSchemaName(), tableName.getTableName()); if (!table.isPresent()) { throw new TableNotFoundException(tableName); } checkTableIsWritable(table.get()); HiveStorageFormat hiveStorageFormat = extractHiveStorageFormat(table.get()); List<HiveColumnHandle> handles = hiveColumnHandles(connectorId, table.get()); return new HiveInsertTableHandle( connectorId, tableName.getSchemaName(), tableName.getTableName(), handles, session.getQueryId(), locationService.forExistingTable(session.getQueryId(), table.get()), hiveStorageFormat); }
private Table getTable(HiveMetastore metastore, SchemaTableName tableName) { Optional<Table> target = metastore.getTable(tableName.getSchemaName(), tableName.getTableName()); if (!target.isPresent()) { throw new TableNotFoundException(tableName); } Table table = target.get(); String protectMode = table.getParameters().get(ProtectMode.PARAMETER_NAME); if (protectMode != null && getProtectModeFromString(protectMode).offline) { throw new TableOfflineException(tableName); } String prestoOffline = table.getParameters().get(PRESTO_OFFLINE); if (!isNullOrEmpty(prestoOffline)) { throw new TableOfflineException(tableName, format("Table '%s' is offline for Presto: %s", tableName, prestoOffline)); } return table; }
public void updateTableLocation(String databaseName, String tableName, String location) { try { Optional<Table> table = getTable(databaseName, tableName); if (!table.isPresent()) { throw new TableNotFoundException(new SchemaTableName(databaseName, tableName)); } table.get().getSd().setLocation(location); try (HiveMetastoreClient client = clientProvider.createMetastoreClient()) { client.alterTable(databaseName, tableName, table.get()); } } catch (TException e) { throw Throwables.propagate(e); } }
protected void dropTable(SchemaTableName table) { try { ConnectorSession session = newSession(); ConnectorTableHandle handle = metadata.getTableHandle(session, table); if (handle == null) { return; } metadata.dropTable(session, handle); try { // todo I have no idea why this is needed... maybe there is a propagation delay in the metastore? metadata.dropTable(session, handle); fail("expected NotFoundException"); } catch (TableNotFoundException expected) { } } catch (Exception e) { Logger.get(getClass()).warn(e, "failed to drop table"); } }
@Override public void dropTable(String databaseName, String tableName) { List<String> locations = listAllDataPaths(this, databaseName, tableName); SchemaTableName schemaTableName = new SchemaTableName(databaseName, tableName); Table table = relations.remove(schemaTableName); if (table == null) { throw new TableNotFoundException(schemaTableName); } views.remove(schemaTableName); partitions.keySet().stream() .filter(partitionName -> partitionName.matches(databaseName, tableName)) .forEach(partitions::remove); // remove data for (String location : locations) { if (location != null) { File directory = new File(URI.create(location)); checkArgument(isParentDir(directory, baseDirectory), "Table directory must be inside of the metastore base directory"); deleteRecursively(directory); } } }
@Override public void alterTable(String databaseName, String tableName, Table newTable) { SchemaTableName oldName = new SchemaTableName(databaseName, tableName); SchemaTableName newName = new SchemaTableName(newTable.getDbName(), newTable.getTableName()); // if the name did not change, this is a simple schema change if (oldName.equals(newName)) { if (relations.replace(oldName, newTable) != null) { throw new TableNotFoundException(oldName); } return; } // remove old table definition and add the new one // TODO: use locking to do this properly Table table = relations.get(oldName); if (table == null) { throw new TableNotFoundException(oldName); } if (relations.putIfAbsent(newName, newTable) != null) { throw new TableAlreadyExistsException(newName); } relations.remove(oldName); }
@Override public void addPartitions(String databaseName, String tableName, List<Partition> partitions) { Optional<Table> table = getTable(databaseName, tableName); if (!table.isPresent()) { throw new TableNotFoundException(new SchemaTableName(databaseName, tableName)); } for (Partition partition : partitions) { String partitionName = createPartitionName(partition, table.get()); partition = partition.deepCopy(); if (partition.getParameters() == null) { partition.setParameters(ImmutableMap.of()); } this.partitions.put(new PartitionName(databaseName, tableName, partitionName), partition); } }
public PRTable getTable(SchemaTableName schemaTableName) throws InterruptedException, ExecutionException, IOException { List<RiakObject> objects = getTableRiakObjects(schemaTableName); //log.info("RiakClient.getTable(%s)", schemaTableName); for (RiakObject o : objects) { //log.debug("ro: %s", o.getValue().toStringUtf8()); PRTable table = objectMapper.readValue(o.getValue().toStringUtf8(), PRTable.class); checkNotNull(table, "table schema (%s) wasn't found.", schemaTableName.getSchemaName()); log.debug("table schema found: %s.", table.getName()); return table; } throw new TableNotFoundException(schemaTableName, "no siblings for " + schemaTableName.toString()); }
@Override public List<JdbcColumnHandle> getColumns(JdbcTableHandle tableHandle) { try (Connection connection = driver.connect(connectionUrl, connectionProperties)) { try (ResultSet resultSet = getColumns(tableHandle, connection.getMetaData())) { List<JdbcColumnHandle> columns = new ArrayList<>(); boolean found = false; while (resultSet.next()) { found = true; Type columnType = toPrestoType(resultSet.getInt("DATA_TYPE")); // skip unsupported column types if (columnType != null) { String columnName = resultSet.getString("COLUMN_NAME"); columns.add(new JdbcColumnHandle(connectorId, columnName, columnType)); } } if (!found) { throw new TableNotFoundException(tableHandle.getSchemaTableName()); } if (columns.isEmpty()) { throw new PrestoException(NOT_SUPPORTED, "Table has no supported column types: " + tableHandle.getSchemaTableName()); } return ImmutableList.copyOf(columns); } } catch (SQLException e) { throw new PrestoException(JDBC_ERROR, e); } }
private void unknownTableColumnHandle(JdbcTableHandle tableHandle) { try { metadata.getColumnHandles(SESSION, tableHandle); fail("Expected getColumnHandle of unknown table to throw a TableNotFoundException"); } catch (TableNotFoundException ignored) { } }
private void unknownTableMetadata(JdbcTableHandle tableHandle) { try { metadata.getTableMetadata(SESSION, tableHandle); fail("Expected getTableMetadata of unknown table to throw a TableNotFoundException"); } catch (TableNotFoundException ignored) { } }
@Test(expectedExceptions = TableNotFoundException.class) public void testInvalidDbGetTable() throws Exception { CassandraTableHandle tableHandle = new CassandraTableHandle(CONNECTOR_ID, BAD_SCHEMA, TEST_TABLE); schemaProvider.getTable(tableHandle); }
private ConnectorTableMetadata getTableMetadata(SchemaTableName tableName) { Optional<Table> table = metastore.getTable(tableName.getSchemaName(), tableName.getTableName()); if (!table.isPresent() || table.get().getTableType().equals(TableType.VIRTUAL_VIEW.name())) { throw new TableNotFoundException(tableName); } Function<HiveColumnHandle, ColumnMetadata> metadataGetter = columnMetadataGetter(table.get(), typeManager); boolean sampled = false; ImmutableList.Builder<ColumnMetadata> columns = ImmutableList.builder(); for (HiveColumnHandle columnHandle : hiveColumnHandles(connectorId, table.get())) { if (columnHandle.getName().equals(SAMPLE_WEIGHT_COLUMN_NAME)) { sampled = true; } else { columns.add(metadataGetter.apply(columnHandle)); } } ImmutableMap.Builder<String, Object> properties = ImmutableMap.builder(); try { HiveStorageFormat format = extractHiveStorageFormat(table.get()); properties.put(STORAGE_FORMAT_PROPERTY, format); } catch (PrestoException ignored) { // todo fail if format is not known } List<String> partitionedBy = table.get().getPartitionKeys().stream() .map(FieldSchema::getName) .collect(toList()); if (!partitionedBy.isEmpty()) { properties.put(PARTITIONED_BY_PROPERTY, partitionedBy); } return new ConnectorTableMetadata(tableName, columns.build(), properties.build(), table.get().getOwner(), sampled); }
@Override public ColumnHandle getSampleWeightColumnHandle(ConnectorSession session, ConnectorTableHandle tableHandle) { SchemaTableName tableName = schemaTableName(tableHandle); Optional<Table> table = metastore.getTable(tableName.getSchemaName(), tableName.getTableName()); if (!table.isPresent()) { throw new TableNotFoundException(tableName); } for (HiveColumnHandle columnHandle : hiveColumnHandles(connectorId, table.get())) { if (columnHandle.getName().equals(SAMPLE_WEIGHT_COLUMN_NAME)) { return columnHandle; } } return null; }
@Override public void renameColumn(ConnectorSession session, ConnectorTableHandle tableHandle, ColumnHandle source, String target) { if (!allowRenameColumn) { throw new PrestoException(PERMISSION_DENIED, "Renaming columns is disabled in this Hive catalog"); } HiveTableHandle hiveTableHandle = checkType(tableHandle, HiveTableHandle.class, "tableHandle"); HiveColumnHandle sourceHandle = checkType(source, HiveColumnHandle.class, "columnHandle"); Optional<Table> tableMetadata = metastore.getTable(hiveTableHandle.getSchemaName(), hiveTableHandle.getTableName()); if (!tableMetadata.isPresent()) { throw new TableNotFoundException(hiveTableHandle.getSchemaTableName()); } Table table = tableMetadata.get(); StorageDescriptor sd = table.getSd(); ImmutableList.Builder<FieldSchema> columns = ImmutableList.builder(); for (FieldSchema fieldSchema : sd.getCols()) { if (fieldSchema.getName().equals(sourceHandle.getName())) { columns.add(new FieldSchema(target, fieldSchema.getType(), fieldSchema.getComment())); } else { columns.add(fieldSchema); } } sd.setCols(columns.build()); table.setSd(sd); metastore.alterTable(hiveTableHandle.getSchemaName(), hiveTableHandle.getTableName(), table); }
@Override public void dropView(ConnectorSession session, SchemaTableName viewName) { ConnectorViewDefinition view = getViews(session, viewName.toSchemaTablePrefix()).get(viewName); if (view == null) { throw new ViewNotFoundException(viewName); } try { metastore.dropTable(viewName.getSchemaName(), viewName.getTableName()); } catch (TableNotFoundException e) { throw new ViewNotFoundException(e.getTableName()); } }
private List<String> getFilteredPartitionNames(HiveMetastore metastore, SchemaTableName tableName, List<HiveColumnHandle> partitionKeys, TupleDomain<ColumnHandle> effectivePredicate) { checkArgument(effectivePredicate.getDomains().isPresent()); List<String> filter = new ArrayList<>(); for (HiveColumnHandle partitionKey : partitionKeys) { Domain domain = effectivePredicate.getDomains().get().get(partitionKey); if (domain != null && domain.isNullableSingleValue()) { Object value = domain.getNullableSingleValue(); if (value == null) { filter.add(HivePartitionKey.HIVE_DEFAULT_DYNAMIC_PARTITION); } else if (value instanceof Slice) { filter.add(((Slice) value).toStringUtf8()); } else if ((value instanceof Boolean) || (value instanceof Double) || (value instanceof Long)) { if (assumeCanonicalPartitionKeys) { filter.add(value.toString()); } else { // Hive treats '0', 'false', and 'False' the same. However, the metastore differentiates between these. filter.add(PARTITION_VALUE_WILDCARD); } } else { throw new PrestoException(NOT_SUPPORTED, "Only Boolean, Double and Long partition keys are supported"); } } else { filter.add(PARTITION_VALUE_WILDCARD); } } // fetch the partition names return metastore.getPartitionNamesByParts(tableName.getSchemaName(), tableName.getTableName(), filter) .orElseThrow(() -> new TableNotFoundException(tableName)); }
@Override public void dropTable(String databaseName, String tableName) { try { Optional<Table> table = getTable(databaseName, tableName); if (!table.isPresent()) { throw new TableNotFoundException(new SchemaTableName(databaseName, tableName)); } // hack to work around the metastore not being configured for S3 List<String> locations = listAllDataPaths(this, databaseName, tableName); table.get().getSd().setLocation("/"); // drop table try (HiveMetastoreClient client = clientProvider.createMetastoreClient()) { client.alterTable(databaseName, tableName, table.get()); client.dropTable(databaseName, tableName, false); } // drop data for (String location : locations) { Path path = new Path(location); hdfsEnvironment.getFileSystem(path).delete(path, true); } } catch (Exception e) { throw Throwables.propagate(e); } finally { invalidateTable(databaseName, tableName); } }
@Override public KinesisTableHandle getTableHandle(ConnectorSession session, SchemaTableName schemaTableName) { KinesisStreamDescription table = getDefinedTables().get(schemaTableName); if (table == null) { throw new TableNotFoundException(schemaTableName); } return new KinesisTableHandle(connectorId, schemaTableName.getSchemaName(), schemaTableName.getTableName(), table.getStreamName(), getDataFormat(table.getMessage())); }
@Override public Map<String, ColumnHandle> getColumnHandles(ConnectorSession connectorSession, ConnectorTableHandle tableHandle) { KinesisTableHandle kinesisTableHandle = handleResolver.convertTableHandle(tableHandle); KinesisStreamDescription kinesisStreamDescription = getDefinedTables().get(kinesisTableHandle.toSchemaTableName()); if (kinesisStreamDescription == null) { throw new TableNotFoundException(kinesisTableHandle.toSchemaTableName()); } ImmutableMap.Builder<String, ColumnHandle> columnHandles = ImmutableMap.builder(); int index = 0; // Note: partition key and related fields are handled by internalFieldDescriptions below KinesisStreamFieldGroup message = kinesisStreamDescription.getMessage(); if (message != null) { List<KinesisStreamFieldDescription> fields = message.getFields(); if (fields != null) { for (KinesisStreamFieldDescription kinesisStreamFieldDescription : fields) { columnHandles.put(kinesisStreamFieldDescription.getName(), kinesisStreamFieldDescription.getColumnHandle(connectorId, index++)); } } } for (KinesisInternalFieldDescription kinesisInternalFieldDescription : internalFieldDescriptions) { columnHandles.put(kinesisInternalFieldDescription.getName(), kinesisInternalFieldDescription.getColumnHandle(connectorId, index++, kinesisConnectorConfig.isHideInternalColumns())); } return columnHandles.build(); }
private List<RiakObject> getTableRiakObjects(SchemaTableName schemaTableName) throws InterruptedException, ExecutionException, IOException { checkNotNull(schemaTableName, "tableName is null"); FetchOperation op = buildFetchOperation( schemaTableName.getSchemaName(), META_BUCKET_NAME, schemaTableName.getTableName()); cluster.execute(op); op.await(); if (!op.isSuccess()) { throw new TableNotFoundException(schemaTableName, op.cause()); } return op.get().getObjectList(); }
public CassandraTable getTable(CassandraTableHandle tableHandle) throws TableNotFoundException { return getCacheValue(tableCache, tableHandle.getSchemaTableName(), TableNotFoundException.class); }
@Test(expectedExceptions = TableNotFoundException.class) public void testGetPartitionsException() throws Exception { metadata.getTableLayouts(newSession(), invalidTableHandle, new Constraint<>(TupleDomain.all(), bindings -> true), Optional.empty()); }
@Test(expectedExceptions = TableNotFoundException.class) public void testGetPartitionNamesException() throws Exception { metadata.getTableLayouts(newSession(), invalidTableHandle, new Constraint<>(TupleDomain.all(), bindings -> true), Optional.empty()); }
@Test(expectedExceptions = TableNotFoundException.class) public void testGetPartitionSplitsBatchInvalidTable() throws Exception { splitManager.getSplits(newSession(), invalidTableLayoutHandle); }