Java 类org.elasticsearch.common.xcontent.support.XContentMapValues 实例源码

/**
 * Waits for the given mapping type to exists on the master node.
 */
public void assertMappingOnMaster(final String index, final String type, final String... fieldNames) throws Exception {
    GetMappingsResponse response = client().admin().indices().prepareGetMappings(index).setTypes(type).get();
    ImmutableOpenMap<String, MappingMetaData> mappings = response.getMappings().get(index);
    assertThat(mappings, notNullValue());
    MappingMetaData mappingMetaData = mappings.get(type);
    assertThat(mappingMetaData, notNullValue());

    Map<String, Object> mappingSource = mappingMetaData.getSourceAsMap();
    assertFalse(mappingSource.isEmpty());
    assertTrue(mappingSource.containsKey("properties"));

    for (String fieldName : fieldNames) {
        Map<String, Object> mappingProperties = (Map<String, Object>) mappingSource.get("properties");
        if (fieldName.indexOf('.') != -1) {
            fieldName = fieldName.replace(".", ".properties.");
        }
        assertThat("field " + fieldName + " doesn't exists in mapping " + mappingMetaData.source().string(), XContentMapValues.extractValue(fieldName, mappingProperties), notNullValue());
    }
}

@Override
public MetadataFieldMapper.Builder parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
    Builder builder = new Builder(parserContext.type());
    for (Iterator<Map.Entry<String, Object>> iterator = node.entrySet().iterator(); iterator.hasNext();) {
        Map.Entry<String, Object> entry = iterator.next();
        String fieldName = entry.getKey();
        Object fieldNode = entry.getValue();
        if (fieldName.equals("type")) {
            builder.type(fieldNode.toString());
            iterator.remove();
        } else if (FIELDDATA.match(fieldName)) {
            // for bw compat only
            Map<String, String> fieldDataSettings = SettingsLoader.Helper.loadNestedFromMap(nodeMapValue(fieldNode, "fielddata"));
            if (fieldDataSettings.containsKey("loading")) {
                builder.eagerGlobalOrdinals("eager_global_ordinals".equals(fieldDataSettings.get("loading")));
            }
            iterator.remove();
        } else if (fieldName.equals("eager_global_ordinals")) {
            builder.eagerGlobalOrdinals(XContentMapValues.nodeBooleanValue(fieldNode, "eager_global_ordinals"));
            iterator.remove();
        }
    }
    return builder;
}

public void testCompletionMultiField() throws Exception {
    assertAcked(
            client().admin().indices().prepareCreate("my-index")
                    .addMapping("my-type", createMappingSource("completion"))
    );

    GetMappingsResponse getMappingsResponse = client().admin().indices().prepareGetMappings("my-index").get();
    MappingMetaData mappingMetaData = getMappingsResponse.mappings().get("my-index").get("my-type");
    assertThat(mappingMetaData, not(nullValue()));
    Map<String, Object> mappingSource = mappingMetaData.sourceAsMap();
    Map aField = ((Map) XContentMapValues.extractValue("properties.a", mappingSource));
    assertThat(aField.size(), equalTo(6));
    assertThat(aField.get("type").toString(), equalTo("completion"));
    assertThat(aField.get("fields"), notNullValue());

    Map bField = ((Map) XContentMapValues.extractValue("properties.a.fields.b", mappingSource));
    assertThat(bField.size(), equalTo(1));
    assertThat(bField.get("type").toString(), equalTo("keyword"));

    client().prepareIndex("my-index", "my-type", "1").setSource("a", "complete me").setRefreshPolicy(IMMEDIATE).get();
    SearchResponse countResponse = client().prepareSearch("my-index").setSize(0).setQuery(matchQuery("a.b", "complete me")).get();
    assertThat(countResponse.getHits().getTotalHits(), equalTo(1L));
}

public void testIpMultiField() throws Exception {
    assertAcked(
            client().admin().indices().prepareCreate("my-index")
                    .addMapping("my-type", createMappingSource("ip"))
    );

    GetMappingsResponse getMappingsResponse = client().admin().indices().prepareGetMappings("my-index").get();
    MappingMetaData mappingMetaData = getMappingsResponse.mappings().get("my-index").get("my-type");
    assertThat(mappingMetaData, not(nullValue()));
    Map<String, Object> mappingSource = mappingMetaData.sourceAsMap();
    Map aField = ((Map) XContentMapValues.extractValue("properties.a", mappingSource));
    assertThat(aField.size(), equalTo(2));
    assertThat(aField.get("type").toString(), equalTo("ip"));
    assertThat(aField.get("fields"), notNullValue());

    Map bField = ((Map) XContentMapValues.extractValue("properties.a.fields.b", mappingSource));
    assertThat(bField.size(), equalTo(1));
    assertThat(bField.get("type").toString(), equalTo("keyword"));

    client().prepareIndex("my-index", "my-type", "1").setSource("a", "127.0.0.1").setRefreshPolicy(IMMEDIATE).get();
    SearchResponse countResponse = client().prepareSearch("my-index").setSize(0).setQuery(matchQuery("a.b", "127.0.0.1")).get();
    assertThat(countResponse.getHits().getTotalHits(), equalTo(1L));
}

public SnapshotsSettings(RiverSettings settings) {
    super();
    if (settings.settings().containsKey(RIVERNAME)) {
        @SuppressWarnings({ "unchecked" })
        Map<String, Object> snapshotterSettings = (Map<String, Object>) settings.settings().get(RIVERNAME);
        this.repository = XContentMapValues.nodeStringValue(snapshotterSettings.get("repository"), "my_backup");
        this.indices = XContentMapValues.nodeStringValue(snapshotterSettings.get("indices"), "_all");
        this.includeGlobalState = XContentMapValues.nodeBooleanValue(snapshotterSettings.get("include_global_state"), false);
        this.frequency = TimeValue.parseTimeValue(XContentMapValues.nodeStringValue(snapshotterSettings.get("frequency"), "24h"), TimeValue.timeValueMinutes(60));
        if (snapshotterSettings.get("purgeAfter") != null && snapshotterSettings.get("purgeAfter").toString().length() > 0) {
            this.purgeAfter = TimeValue.parseTimeValue(XContentMapValues.nodeStringValue(snapshotterSettings.get("purgeAfter"), "240h"), TimeValue.timeValueHours(240));
        } else {
            this.purgeAfter = null;
        }
        this.setPurgeIndicesMustMatch(XContentMapValues.nodeBooleanValue(snapshotterSettings.get("purge_indices_must_match"), true));

    } else {
        this.repository = "my_backup";
        this.indices = "_all";
        this.includeGlobalState = false;
        this.frequency = TimeValue.timeValueHours(24);
        this.purgeAfter = null; // no purging by default
        this.setPurgeIndicesMustMatch(true);
    }
}

@SuppressWarnings({"unchecked", "rawtypes"})
@Override
public Mapper.Builder parse(String name, Map<String, Object> mapping, ParserContext parserContext) {
    StandardnumberMapper.Builder builder = new Builder(name, service);
    Iterator<Map.Entry<String, Object>> iterator = mapping.entrySet().iterator();
    while (iterator.hasNext()) {
        Map.Entry<String, Object> entry = iterator.next();
        String fieldName = entry.getKey();
        Object fieldNode = entry.getValue();
        switch (fieldName) {
            case "standardnumbers" :
                builder.settingsBuilder.putArray("standardnumbers", XContentMapValues.nodeStringArrayValue(fieldNode));
                iterator.remove();
                break;
            default:
                break;
        }
    }
    return builder;
}

private Feeder createFeeder(String riverType, String riverName, RiverSettings riverSettings) {
    JDBCFeeder feeder = null;
    try {
        Map<String, Object> spec = (Map<String, Object>) riverSettings.settings().get("jdbc");
        Map<String, String> loadedSettings = new JsonSettingsLoader().load(jsonBuilder().map(spec).string());
        Settings mySettings = settingsBuilder().put(loadedSettings).build();
        String strategy = XContentMapValues.nodeStringValue(spec.get("strategy"), "simple");
        RiverFlow riverFlow = RiverServiceLoader.findRiverFlow(strategy);
        logger.debug("found river flow class {} for strategy {}", riverFlow.getClass().getName(), strategy);
        feeder = riverFlow.getFeeder();
        logger.debug("spec = {} settings = {}", spec, mySettings.getAsMap());
        feeder.setName(riverName)
                .setType(riverType)
                .setSpec(spec).setSettings(mySettings);
    } catch (IOException e) {
        logger.error(e.getMessage(), e);
    }
    return feeder;
}

@Override
protected void createRiverContext(String riverType, String riverName, Map<String, Object> mySettings) throws IOException {
    super.createRiverContext(riverType, riverName, mySettings);
    // defaults for column strategy
    String columnCreatedAt = XContentMapValues.nodeStringValue(mySettings.get("created_at"), "created_at");
    String columnUpdatedAt = XContentMapValues.nodeStringValue(mySettings.get("updated_at"), "updated_at");
    String columnDeletedAt = XContentMapValues.nodeStringValue(mySettings.get("deleted_at"), null);
    boolean columnEscape = XContentMapValues.nodeBooleanValue(mySettings.get("column_escape"), true);
    TimeValue lastRunTimeStampOverlap = XContentMapValues.nodeTimeValue(mySettings.get("last_run_timestamp_overlap"),
            TimeValue.timeValueSeconds(0));
    riverContext
            .columnCreatedAt(columnCreatedAt)
            .columnUpdatedAt(columnUpdatedAt)
            .columnDeletedAt(columnDeletedAt)
            .columnEscape(columnEscape)
            .setLastRunTimeStampOverlap(lastRunTimeStampOverlap);
}

@Test
public void testAttachments() throws Exception{

        Map<String, Object> settings = settings("/river-imap-attachments.json");

    final Properties props = new Properties();
    final String user = XContentMapValues.nodeStringValue(settings.get("user"), null);
    final String password = XContentMapValues.nodeStringValue(settings.get("password"), null);

    for (final Map.Entry<String, Object> entry : settings.entrySet()) {

        if (entry != null && entry.getKey().startsWith("mail.")) {
            props.setProperty(entry.getKey(), String.valueOf(entry.getValue()));
        }
    }

    registerRiver("imap_river", "river-imap-attachments.json");

    final Session session = Session.getInstance(props);
    final Store store = session.getStore();
    store.connect(user, password);
    checkStoreForTestConnection(store);
    final Folder inbox = store.getFolder("INBOX");
    inbox.open(Folder.READ_WRITE);



    final MimeMessage message = new MimeMessage(session);
    message.setFrom(new InternetAddress(EMAIL_TO));
    message.addRecipient(Message.RecipientType.TO, new InternetAddress(EMAIL_USER_ADDRESS));
    message.setSubject(EMAIL_SUBJECT + "::attachment test");
    message.setSentDate(new Date());

    BodyPart bp = new MimeBodyPart();
    bp.setText("Text");
    Multipart mp = new MimeMultipart();
    mp.addBodyPart(bp);

    bp = new MimeBodyPart();
    DataSource ds = new ByteArrayDataSource(this.getClass().getResourceAsStream("/httpclient-tutorial.pdf"), AttachmentMapperTest.APPLICATION_PDF);
    bp.setDataHandler(new DataHandler(ds));
    bp.setFileName("httpclient-tutorial.pdf");
    mp.addBodyPart(bp);
    message.setContent(mp);

    inbox.appendMessages(new Message[]{message});
    IMAPUtils.close(inbox);
    IMAPUtils.close(store);

    //let the river index
    Thread.sleep(20*1000);

    esSetup.client().admin().indices().refresh(new RefreshRequest()).actionGet();

    SearchResponse searchResponse =  esSetup.client().prepareSearch("imapriverdata").setTypes("mail").execute().actionGet();
    Assert.assertEquals(1, searchResponse.getHits().totalHits());

    //BASE64 content httpclient-tutorial.pdf
    Assert.assertTrue(searchResponse.getHits().hits()[0].getSourceAsString().contains(AttachmentMapperTest.PDF_BASE64_DETECTION));

    searchResponse =  esSetup.client().prepareSearch("imapriverdata").addFields("*").setTypes("mail").setQuery(QueryBuilders.matchPhraseQuery("attachments.content.content", PDF_CONTENT_TO_SEARCH)).execute().actionGet();
    Assert.assertEquals(1, searchResponse.getHits().totalHits());

    Assert.assertEquals(1, searchResponse.getHits().hits()[0].field("attachments.content.content").getValues().size());
    Assert.assertEquals("HttpClient Tutorial", searchResponse.getHits().hits()[0].field("attachments.content.title").getValue().toString());
    Assert.assertEquals("application/pdf", searchResponse.getHits().hits()[0].field("attachments.content.content_type").getValue().toString());
    Assert.assertTrue(searchResponse.getHits().hits()[0].field("attachments.content.content").getValue().toString().contains(PDF_CONTENT_TO_SEARCH));

}

private boolean isStarted(){
   // Refresh index before querying it.
   client.admin().indices().prepareRefresh("_river").execute().actionGet();
   GetResponse isStartedGetResponse = client.prepareGet("_river", riverName().name(), "_s3status").execute().actionGet();
   try{
      if (!isStartedGetResponse.isExists()){
         XContentBuilder xb = jsonBuilder().startObject()
               .startObject("amazon-s3")
                  .field("feedname", feedDefinition.getFeedname())
                  .field("status", "STARTED").endObject()
               .endObject();
         client.prepareIndex("_river", riverName.name(), "_s3status").setSource(xb).execute();
         return true;
      } else {
         String status = (String)XContentMapValues.extractValue("amazon-s3.status", isStartedGetResponse.getSourceAsMap());
         if ("STOPPED".equals(status)){
            return false;
         }
      }
   } catch (Exception e){
      logger.warn("failed to get status for " + riverName().name() + ", throttling....", e);
   }
   return true;
}

/**
 * Extract array from settings (array or ; delimited String)
 * @param settings Settings
 * @param path Path to settings definition
 * @return Array of settings
 */
@SuppressWarnings("unchecked")
public static String[] buildArrayFromSettings(Map<String, Object> settings, String path){
   String[] includes;

   // We manage comma separated format and arrays
   if (XContentMapValues.isArray(XContentMapValues.extractValue(path, settings))) {
      List<String> includesarray = (List<String>) XContentMapValues.extractValue(path, settings);
      int i = 0;
      includes = new String[includesarray.size()];
      for (String include : includesarray) {
         includes[i++] = trimAllWhitespace(include);
      }
   } else {
      String includedef = (String) XContentMapValues.extractValue(path, settings);
      includes = Strings.commaDelimitedListToStringArray(trimAllWhitespace(includedef));
   }

   String[] uniquelist = removeDuplicateStrings(includes);

   return uniquelist;
}

/**
 * Extract array from settings (array or ; delimited String)
 * @param settings Settings
 * @param path Path to settings definition
 * @return Array of settings
 */
@SuppressWarnings("unchecked")
public static String[] buildArrayFromSettings(Map<String, Object> settings, String path){
   String[] includes;

   // We manage comma separated format and arrays
   if (XContentMapValues.isArray(XContentMapValues.extractValue(path, settings))) {
      List<String> includesarray = (List<String>) XContentMapValues.extractValue(path, settings);
      int i = 0;
      includes = new String[includesarray.size()];
      for (String include : includesarray) {
         includes[i++] = Strings.trimAllWhitespace(include);
      }
   } else {
      String includedef = (String) XContentMapValues.extractValue(path, settings);
      includes = Strings.commaDelimitedListToStringArray(Strings.trimAllWhitespace(includedef));
   }

   String[] uniquelist = Strings.removeDuplicateStrings(includes);

   return uniquelist;
}

private boolean isStarted(){
   // Refresh index before querying it.
   client.admin().indices().prepareRefresh("_river").execute().actionGet();
   GetResponse isStartedGetResponse = client.prepareGet("_river", riverName().name(), "_drivestatus").execute().actionGet();
   try{
      if (!isStartedGetResponse.isExists()){
         XContentBuilder xb = jsonBuilder().startObject()
               .startObject("google-drive")
                  .field("feedname", feedDefinition.getFeedname())
                  .field("status", "STARTED").endObject()
               .endObject();
         client.prepareIndex("_river", riverName.name(), "_drivestatus").setSource(xb).execute();
         return true;
      } else {
         String status = (String)XContentMapValues.extractValue("google-drive.status", isStartedGetResponse.getSourceAsMap());
         if ("STOPPED".equals(status)){
            return false;
         }
      }
   } catch (Exception e){
      logger.warn("failed to get status for " + riverName().name() + ", throttling....", e);
   }
   return true;
}

private void parseConfig() {
    if (settings.settings().containsKey("wildfly")) {
        Map<String, Object> wildflySettings = (Map<String, Object>) settings.settings().get("wildfly");

        this.username = XContentMapValues.nodeStringValue(wildflySettings.get("user"), null);
        this.password = XContentMapValues.nodeStringValue(wildflySettings.get("user"), null);
        this.host = XContentMapValues.nodeStringValue(wildflySettings.get("host"), null);
        this.port = XContentMapValues.nodeIntegerValue(wildflySettings.get("port"), 9999);
        this.scheduleSeconds = XContentMapValues.nodeIntegerValue(wildflySettings.get("schedule"), 1);

    }
    else
    {
        logger.error("invalid wildfly plugin configuration");
    }

    if (settings.settings().containsKey("index")) {
        Map<String, Object> indexSettings = (Map<String, Object>) settings.settings().get("index");
        indexName = XContentMapValues.nodeStringValue(indexSettings.get("index"), riverName.name());
        typeName = XContentMapValues.nodeStringValue(indexSettings.get("type"), "status");
    }
    else
    {
        logger.error("invalid wildfly plugin configuration");
    }
}

@SuppressWarnings("unchecked")
private List<Attribute> nodeToAttributeList(Object obj, List<Attribute> defaultValue) {
    if (null != obj && XContentMapValues.isArray(obj)) {
        List<Attribute> res = new ArrayList<Attribute>();
        for (Object o : (List<Object>) obj) {
            if (o instanceof String) {
                res.add(new Attribute(o.toString()));
            } else if (o instanceof Map) {
                Map<String, Object> oMap = (Map<String, Object>) o;
                if (oMap.containsKey("name")) {
                    if (oMap.containsKey("transform")) {
                        res.add(new Attribute(oMap.get("name").toString(), oMap.get("transform").toString()));
                    } else {
                        res.add(new Attribute(oMap.get("name").toString()));
                    }
                }               
            }
        }
        return res;
    }
    return defaultValue;
   }

private void loadDefaultsIfNecessary() {
    Map<String, Object> settingsDefault = loadDefaultSettingsMapFromFile();

    indexFieldForRiverName = loadDefaultStringIfNecessary(indexFieldForRiverName, CONFIG_FIELDRIVERNAME,
            settingsDefault);
    indexFieldForSpaceKey = loadDefaultStringIfNecessary(indexFieldForSpaceKey, CONFIG_FIELDSPACEKEY, settingsDefault);
    indexFieldForRemoteDocumentId = loadDefaultStringIfNecessary(indexFieldForRemoteDocumentId, CONFIG_FIELDDOCUMENTID,
            settingsDefault);

    if (filtersConfig == null)
        filtersConfig = new HashMap<String, Map<String, String>>();

    if (commentIndexingMode == null) {
        commentIndexingMode = CommentIndexingMode.parseConfiguration(XContentMapValues.nodeStringValue(
                settingsDefault.get(CONFIG_COMMENTMODE), null));
    }
}

@Override
public Date extractDocumentUpdated(Map<String, Object> document) {
    Object val = XContentMapValues.extractValue(remoteDataFieldForUpdated, document);
    if (val == null)
        return null;
    if (!Utils.isSimpleValue(val))
        throw new SettingsException("Remote data field '" + remoteDataFieldForUpdated
                + "' must provide simple value, but value is " + val);

    if (val instanceof Date)
        return (Date) val;

    try {
        return DateTimeUtils.parseDate( val.toString(), remoteDataFieldForUpdatedFormat );
    } catch (IllegalArgumentException e1) {
        throw new SettingsException("Remote data field '" + remoteDataFieldForUpdated
                + "' is not reecognized: " + val);
    }
}

@Override
public boolean extractDocumentDeleted(Map<String, Object> document) {
    if (document == null || remoteDataFieldForDeleted == null)
        return false;

    Object val = XContentMapValues.extractValue(remoteDataFieldForDeleted, document);
    if (val == null)
        return false;

    if (!Utils.isSimpleValue(val))
        throw new SettingsException("Remote data field '" + remoteDataFieldForDeleted
                + "' must provide simple value, but value is " + val);

    String v = null;
    if (val instanceof String) {
        v = (String) val;
    } else {
        v = val.toString();
    }

    return v.equals(remoteDataValueForDeleted);
}

@Override
public HashMap<String, String> loadKey(String username) {
    logger.info("loading password for username {}", username);
    HashMap<String, String> ret = new HashMap<>();
    String riverIndexName = getRiverIndexName();
    refreshSearchIndex(riverIndexName);
    GetResponse resp = client.prepareGet(riverIndexName, riverName().name(), "_pwd").execute().actionGet();
    if (resp.isExists()) {
        if (logger.isDebugEnabled()) {
            logger.debug("Password document: {}", resp.getSourceAsString());
        }
        Map<String, Object> newset = resp.getSource();
        Set<String> keys = newset.keySet();
        for(String s : keys){
            logger.info("Added key {} with a value of {}",s,XContentMapValues.nodeStringValue(newset.get(s), null));
            ret.put(s,XContentMapValues.nodeStringValue(newset.get(s), null));

        }

    }
    if(ret.isEmpty()){
        return null;
    }
    return ret;
}

/**
 * Get url from configuration and validate it for format, and for presence.
 * 
 * @param config to get URL from
 * @param cfgProperyName name of config property with URL
 * @param mandatory if URL is mandatory so validation is performed
 * @return url
 * @throws SettingsException in case of validation error
 */
protected static String getUrlFromConfig(Map<String, Object> config, String cfgProperyName, boolean mandatory)
        throws SettingsException {
    String url = null;
    if (config != null)
        url = Utils.trimToNull(XContentMapValues.nodeStringValue(config.get(cfgProperyName), null));
    if (mandatory && url == null) {
        throw new SettingsException("remote/" + cfgProperyName + " element of configuration structure not found or empty");
    }
    if (url != null) {
        try {
            new URL(url);
        } catch (MalformedURLException e) {
            throw new SettingsException("Parameter remote/" + cfgProperyName + " is malformed URL " + e.getMessage());
        }
    }
    return url;
}

/**
 * Returns a filter function that expects the source map as an input and returns
 * the filtered map.
 */
public Function<Map<String, ?>, Map<String, Object>> getFilter() {
    if (filter == null) {
        filter = XContentMapValues.filter(includes, excludes);
    }
    return filter;
}

private List<Object> fetch(TermsLookup termsLookup, Client client) {
    List<Object> terms = new ArrayList<>();
    GetRequest getRequest = new GetRequest(termsLookup.index(), termsLookup.type(), termsLookup.id())
            .preference("_local").routing(termsLookup.routing());
    final GetResponse getResponse = client.get(getRequest).actionGet();
    if (getResponse.isSourceEmpty() == false) { // extract terms only if the doc source exists
        List<Object> extractedValues = XContentMapValues.extractRawValues(termsLookup.path(), getResponse.getSourceAsMap());
        terms.addAll(extractedValues);
    }
    return terms;
}

private SourceFieldMapper(boolean enabled, String[] includes, String[] excludes, Settings indexSettings) {
    super(NAME, Defaults.FIELD_TYPE.clone(), Defaults.FIELD_TYPE, indexSettings); // Only stored.
    this.enabled = enabled;
    this.includes = includes;
    this.excludes = excludes;
    final boolean filtered = (includes != null && includes.length > 0) || (excludes != null && excludes.length > 0);
    this.filter = enabled && filtered && fieldType().stored() ? XContentMapValues.filter(includes, excludes) : null;
    this.complete = enabled && includes == null && excludes == null;
}

@Override
public Mapper.Builder<?,?> parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
    KeywordFieldMapper.Builder builder = new KeywordFieldMapper.Builder(name);
    parseField(builder, name, node, parserContext);
    for (Iterator<Map.Entry<String, Object>> iterator = node.entrySet().iterator(); iterator.hasNext();) {
        Map.Entry<String, Object> entry = iterator.next();
        String propName = entry.getKey();
        Object propNode = entry.getValue();
        if (propName.equals("null_value")) {
            if (propNode == null) {
                throw new MapperParsingException("Property [null_value] cannot be null.");
            }
            builder.nullValue(propNode.toString());
            iterator.remove();
        } else if (propName.equals("ignore_above")) {
            builder.ignoreAbove(XContentMapValues.nodeIntegerValue(propNode, -1));
            iterator.remove();
        } else if (propName.equals("norms")) {
            builder.omitNorms(XContentMapValues.nodeBooleanValue(propNode, "norms") == false);
            iterator.remove();
        } else if (propName.equals("eager_global_ordinals")) {
            builder.eagerGlobalOrdinals(XContentMapValues.nodeBooleanValue(propNode, "eager_global_ordinals"));
            iterator.remove();
        } else if (propName.equals("normalizer")) {
            if (propNode != null) {
                NamedAnalyzer normalizer = parserContext.getIndexAnalyzers().getNormalizer(propNode.toString());
                if (normalizer == null) {
                    throw new MapperParsingException("normalizer [" + propNode.toString() + "] not found for field [" + name + "]");
                }
                builder.normalizer(normalizer);
            }
            iterator.remove();
        }
    }
    return builder;
}

@Override
public Mapper.Builder parse(String fieldName, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
    TextFieldMapper.Builder builder = new TextFieldMapper.Builder(fieldName);
    builder.fieldType().setIndexAnalyzer(parserContext.getIndexAnalyzers().getDefaultIndexAnalyzer());
    builder.fieldType().setSearchAnalyzer(parserContext.getIndexAnalyzers().getDefaultSearchAnalyzer());
    builder.fieldType().setSearchQuoteAnalyzer(parserContext.getIndexAnalyzers().getDefaultSearchQuoteAnalyzer());
    parseTextField(builder, fieldName, node, parserContext);
    for (Iterator<Map.Entry<String, Object>> iterator = node.entrySet().iterator(); iterator.hasNext();) {
        Map.Entry<String, Object> entry = iterator.next();
        String propName = entry.getKey();
        Object propNode = entry.getValue();
        if (propName.equals("position_increment_gap")) {
            int newPositionIncrementGap = XContentMapValues.nodeIntegerValue(propNode, -1);
            builder.positionIncrementGap(newPositionIncrementGap);
            iterator.remove();
        } else if (propName.equals("fielddata")) {
            builder.fielddata(XContentMapValues.nodeBooleanValue(propNode, "fielddata"));
            iterator.remove();
        } else if (propName.equals("eager_global_ordinals")) {
            builder.eagerGlobalOrdinals(XContentMapValues.nodeBooleanValue(propNode, "eager_global_ordinals"));
            iterator.remove();
        } else if (propName.equals("fielddata_frequency_filter")) {
            Map<?,?> frequencyFilter = (Map<?, ?>) propNode;
            double minFrequency = XContentMapValues.nodeDoubleValue(frequencyFilter.remove("min"), 0);
            double maxFrequency = XContentMapValues.nodeDoubleValue(frequencyFilter.remove("max"), Integer.MAX_VALUE);
            int minSegmentSize = XContentMapValues.nodeIntegerValue(frequencyFilter.remove("min_segment_size"), 0);
            builder.fielddataFrequencyFilter(minFrequency, maxFrequency, minSegmentSize);
            DocumentMapperParser.checkNoRemainingFields(propName, frequencyFilter, parserContext.indexVersionCreated());
            iterator.remove();
        }
    }
    return builder;
}

public static boolean nodeBooleanValue(String fieldName, String propertyName, Object node,
                                       Mapper.TypeParser.ParserContext parserContext) {
    if (parserContext.indexVersionCreated().onOrAfter(Version.V_6_0_0_alpha1_UNRELEASED)) {
        return XContentMapValues.nodeBooleanValue(node, fieldName + "." + propertyName);
    } else {
        return nodeBooleanValueLenient(fieldName, propertyName, node);
    }
}

public void testGeoPointMultiField() throws Exception {
    assertAcked(
            client().admin().indices().prepareCreate("my-index")
                    .addMapping("my-type", createMappingSource("geo_point"))
    );

    GetMappingsResponse getMappingsResponse = client().admin().indices().prepareGetMappings("my-index").get();
    MappingMetaData mappingMetaData = getMappingsResponse.mappings().get("my-index").get("my-type");
    assertThat(mappingMetaData, not(nullValue()));
    Map<String, Object> mappingSource = mappingMetaData.sourceAsMap();
    Map aField = ((Map) XContentMapValues.extractValue("properties.a", mappingSource));
    logger.info("Keys: {}", aField.keySet());
    assertThat(aField.size(), equalTo(2));
    assertThat(aField.get("type").toString(), equalTo("geo_point"));
    assertThat(aField.get("fields"), notNullValue());

    Map bField = ((Map) XContentMapValues.extractValue("properties.a.fields.b", mappingSource));
    assertThat(bField.size(), equalTo(1));
    assertThat(bField.get("type").toString(), equalTo("keyword"));

    GeoPoint point = new GeoPoint(51, 19);
    client().prepareIndex("my-index", "my-type", "1").setSource("a", point.toString()).setRefreshPolicy(IMMEDIATE).get();
    SearchResponse countResponse = client().prepareSearch("my-index").setSize(0)
            .setQuery(constantScoreQuery(geoDistanceQuery("a").point(51, 19).distance(50, DistanceUnit.KILOMETERS)))
            .get();
    assertThat(countResponse.getHits().getTotalHits(), equalTo(1L));
    countResponse = client().prepareSearch("my-index").setSize(0).setQuery(matchQuery("a.b", point.geohash())).get();
    assertThat(countResponse.getHits().getTotalHits(), equalTo(1L));
}

public void testTokenCountMultiField() throws Exception {
    assertAcked(
            client().admin().indices().prepareCreate("my-index")
                    .addMapping("my-type", XContentFactory.jsonBuilder().startObject().startObject("my-type")
                            .startObject("properties")
                            .startObject("a")
                            .field("type", "token_count")
                            .field("analyzer", "simple")
                            .startObject("fields")
                            .startObject("b")
                            .field("type", "keyword")
                            .endObject()
                            .endObject()
                            .endObject()
                            .endObject()
                            .endObject().endObject())
    );

    GetMappingsResponse getMappingsResponse = client().admin().indices().prepareGetMappings("my-index").get();
    MappingMetaData mappingMetaData = getMappingsResponse.mappings().get("my-index").get("my-type");
    assertThat(mappingMetaData, not(nullValue()));
    Map<String, Object> mappingSource = mappingMetaData.sourceAsMap();
    Map aField = ((Map) XContentMapValues.extractValue("properties.a", mappingSource));
    assertThat(aField.size(), equalTo(3));
    assertThat(aField.get("type").toString(), equalTo("token_count"));
    assertThat(aField.get("fields"), notNullValue());

    Map bField = ((Map) XContentMapValues.extractValue("properties.a.fields.b", mappingSource));
    assertThat(bField.size(), equalTo(1));
    assertThat(bField.get("type").toString(), equalTo("keyword"));

    client().prepareIndex("my-index", "my-type", "1").setSource("a", "my tokens").setRefreshPolicy(IMMEDIATE).get();
    SearchResponse countResponse = client().prepareSearch("my-index").setSize(0).setQuery(matchQuery("a.b", "my tokens")).get();
    assertThat(countResponse.getHits().getTotalHits(), equalTo(1L));
}

@Nullable
@Override
public Object referenceValue(Reference reference) {
    if (updatedColumnValues == null) {
        return super.referenceValue(reference);
    }

    Object value = updatedColumnValues.get(reference.ident().columnIdent().fqn());
    if (value == null && !reference.ident().isColumn()) {
        value = XContentMapValues.extractValue(reference.ident().columnIdent().fqn(), updatedColumnValues);
    }
    return reference.valueType().value(value);
}

@Override
public Function<GetResult, Object> build(final Reference reference, SymbolToFieldExtractor.Context context) {
    return new Function<GetResult, Object>() {
        @Override
        public Object apply(GetResult getResult) {
            if (getResult == null) {
                return null;
            }
            return reference.valueType().value(XContentMapValues.extractValue(
                    reference.info().ident().columnIdent().fqn(), getResult.sourceAsMap()));
        }
    };
}

@Override
public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
    Builder builder = geoShapeField(name);
    for (Iterator<Map.Entry<String, Object>> iterator = node.entrySet().iterator(); iterator.hasNext();) {
        Map.Entry<String, Object> entry = iterator.next();
        String fieldName = Strings.toUnderscoreCase(entry.getKey());
        Object fieldNode = entry.getValue();
        if (Names.TREE.equals(fieldName)) {
            builder.fieldType().setTree(fieldNode.toString());
            iterator.remove();
        } else if (Names.TREE_LEVELS.equals(fieldName)) {
            builder.fieldType().setTreeLevels(Integer.parseInt(fieldNode.toString()));
            iterator.remove();
        } else if (Names.TREE_PRESISION.equals(fieldName)) {
            builder.fieldType().setPrecisionInMeters(DistanceUnit.parse(fieldNode.toString(), DistanceUnit.DEFAULT, DistanceUnit.DEFAULT));
            iterator.remove();
        } else if (Names.DISTANCE_ERROR_PCT.equals(fieldName)) {
            builder.fieldType().setDistanceErrorPct(Double.parseDouble(fieldNode.toString()));
            iterator.remove();
        } else if (Names.ORIENTATION.equals(fieldName)) {
            builder.fieldType().setOrientation(ShapeBuilder.orientationFromString(fieldNode.toString()));
            iterator.remove();
        } else if (Names.STRATEGY.equals(fieldName)) {
            builder.fieldType().setStrategyName(fieldNode.toString());
            iterator.remove();
        } else if (Names.COERCE.equals(fieldName)) {
            builder.coerce(nodeBooleanValue(fieldNode));
            iterator.remove();
        } else if (Names.STRATEGY_POINTS_ONLY.equals(fieldName)
            && builder.fieldType().strategyName.equals(SpatialStrategy.TERM.getStrategyName()) == false) {
            builder.fieldType().setPointsOnly(XContentMapValues.nodeBooleanValue(fieldNode));
            iterator.remove();
        }
    }
    return builder;
}

@Override
public ExecutableScript newScript(@Nullable Map<String, Object> params) {
    String fieldName = params == null ? null : XContentMapValues.nodeStringValue(params.get("field"), null);
    String hashValue = params == null ? null : XContentMapValues.nodeStringValue(params.get("hash"), null);

    if (fieldName == null) {
        throw new ScriptException("Missing the field parameter");
    }
    return new HammingDistanceScript(fieldName, hashValue);
}

public ExecutableScript newScript(@Nullable Map<String, Object> params) {

        String pattern = params == null ? "" :
                XContentMapValues.nodeStringValue(params.get("pattern"), "");

        String fieldName = params == null ? "" :
                XContentMapValues.nodeStringValue(params.get("fieldName"), "");

        String groupkeys = params == null ? "" :
                XContentMapValues.nodeStringValue(params.get("groupkeys"), "");

        String isHashMap = params == null ? "" :
                XContentMapValues.nodeStringValue(params.get("isHashMap"), "");


        if (StringUtils.isBlank(fieldName)) {
            throw new ScriptException("Missing field parameter");
        }
        if (StringUtils.isBlank(pattern)) {
            throw new ScriptException("Missing field parameter");
        }

        List<String> groupkeyList = new ArrayList<>();
        if (StringUtils.isNotBlank(groupkeys)) {
            groupkeyList = Arrays.asList(groupkeys.split(","));
        }

        Boolean isHashMapBoolean;
        if (StringUtils.isBlank(isHashMap) || "false".equals(isHashMap.toLowerCase())) {
            isHashMapBoolean = false;
        } else {
            isHashMapBoolean = true;
        }

        return new GrokNativeScript(pattern, fieldName, groupkeyList, isHashMapBoolean);
    }

@Override
public ExecutableScript newScript(@Nullable Map<String, Object> params) {
    if (params == null || params.containsKey("spec") == false) {
        throw new IllegalArgumentException("the spec parameter is required");
    }
    Map<String, Object> spec = XContentMapValues.nodeMapValue(params.get("spec"), "spec");
    // TODO: Add caching mechanism
    VectorRangesToVector features = new VectorRangesToVectorJSON(spec);
    return new VectorizerScript(features);
}

/**
 * Waits for the given mapping type to exists on the master node.
 */
public void waitForMappingOnMaster(final String index, final String type, final String... fieldNames) throws Exception {
    assertBusy(new Callable() {
        @Override
        public Object call() throws Exception {
            final GetMappingsResponse response = client().admin().indices().prepareGetMappings(index).setTypes(type).get();
            final ImmutableOpenMap<String, MappingMetaData> mappings = response.getMappings().get(index);
            assertThat(mappings, notNullValue());
            final MappingMetaData mappingMetaData = mappings.get(type);
            assertThat(mappingMetaData, notNullValue());

            final Map<String, Object> mappingSource = mappingMetaData.getSourceAsMap();
            assertFalse(mappingSource.isEmpty());
            assertTrue(mappingSource.containsKey("properties"));

            for (String fieldName : fieldNames) {
                final Map<String, Object> mappingProperties = (Map<String, Object>) mappingSource.get("properties");
                if (fieldName.indexOf('.') != -1) {
                    fieldName = fieldName.replace(".", ".properties.");
                }
                assertThat("field " + fieldName + " doesn't exists in mapping " + mappingMetaData.source().string(),
                        XContentMapValues.extractValue(fieldName, mappingProperties), notNullValue());
            }

            return null;
        }
    });
}

@SuppressWarnings({ "unchecked" })
private String readConfig(final String config, final String defaultValue) {
    if (settings.settings().containsKey("bigquery")) {
        Map<String, Object> bqSettings = (Map<String, Object>) settings.settings().get("bigquery");
        return XContentMapValues.nodeStringValue(bqSettings.get(config), defaultValue);
    }
    return defaultValue;
}

@SuppressWarnings({"unchecked"})
@Inject
public GitHubRiver(RiverName riverName, RiverSettings settings, Client client) {
    super(riverName, settings);
    this.client = client;

    if (!settings.settings().containsKey("github")) {
        throw new IllegalArgumentException("Need river settings - owner and repository.");
    }

    // get settings
    Map<String, Object> githubSettings = (Map<String, Object>) settings.settings().get("github");
    owner = XContentMapValues.nodeStringValue(githubSettings.get("owner"), null);
    repository = XContentMapValues.nodeStringValue(githubSettings.get("repository"), null);

    index = String.format("%s&%s", owner, repository);
    userRequestedInterval = XContentMapValues.nodeIntegerValue(githubSettings.get("interval"), 60);

    // auth (optional)
    username = null;
    password = null;
    if (githubSettings.containsKey("authentication")) {
        Map<String, Object> auth = (Map<String, Object>) githubSettings.get("authentication");
        username = XContentMapValues.nodeStringValue(auth.get("username"), null);
        password = XContentMapValues.nodeStringValue(auth.get("password"), null);
    }

    // endpoint (optional - default to github.com)
    endpoint = XContentMapValues.nodeStringValue(githubSettings.get("endpoint"), "https://api.github.com");

    logger.info("Created GitHub river.");
}

/**
 * This method is called for every search on every shard.
 *
 * @param params list of script parameters passed with the query
 * @return new native script
 */
@Override
public ExecutableScript newScript(@Nullable Map<String, Object> params) {
    if (params == null) {
        throw new ScriptException("Missing script parameters");
    }
    // stoic specific concept. if the name of the index for the hit is "b-something"
    // and no index is defined, we generate the lookup index by replacing the hit's index "b-" by "u-$tenant-"
    String tenant = XContentMapValues.nodeStringValue(params.get("tenant"), null);

    String index = XContentMapValues.nodeStringValue(params.get("index"), null);
    String type = XContentMapValues.nodeStringValue(params.get("type"), null);
    String id = XContentMapValues.nodeStringValue(params.get("id"), null);
    String sourceIndex = XContentMapValues.nodeStringValue(params.get("source_index"), null);
    String sourceType = XContentMapValues.nodeStringValue(params.get("source_type"), null);
    String sourceId = XContentMapValues.nodeStringValue(params.get("source_id"), null);
    String targetSource = XContentMapValues.nodeStringValue(params.get("target_source"), null);
    String targetSourceIncludes = XContentMapValues.nodeStringValue(params.get("target_source_includes"), null);
    String targetSourceExcludes = XContentMapValues.nodeStringValue(params.get("target_source_excludes"), null);
    String targetSourceLwc = XContentMapValues.nodeStringValue(params.get("target_source_lwc"), null);
    String missing = XContentMapValues.nodeStringValue(params.get("missing"), null);
    String cacheFieldSave = XContentMapValues.nodeStringValue(params.get("cache_field_save"), null); // write a cache field on the document.
    String cachedFieldRead = XContentMapValues.nodeStringValue(params.get("cached_field"), null);
    return new JoinerScript(node.client(), logger, cache,
            tenant, index, type, id,
            sourceIndex, sourceType, sourceId,
            targetSource, targetSourceIncludes, targetSourceExcludes,
            targetSourceLwc, missing,
            cacheFieldSave, cachedFieldRead);
}

@SuppressWarnings({"unchecked"})
public static List<SQLCommand> parse(Map<String, Object> settings) {
    List<SQLCommand> sql = new LinkedList<SQLCommand>();
    if (!XContentMapValues.isArray(settings.get("sql"))) {
        settings.put("sql", Arrays.asList(settings.get("sql")));
    }
    List<Object> list = (List<Object>) settings.get("sql");
    for (Object entry : list) {
        SQLCommand command = new SQLCommand();
        try {
            if (entry instanceof Map) {
                Map<String, Object> m = (Map<String, Object>) entry;
                if (m.containsKey("statement")) {
                    command.setSQL((String) m.get("statement"));
                }
                if (m.containsKey("parameter")) {
                    command.setParameters(XContentMapValues.extractRawValues("parameter", m));
                }
                if (m.containsKey("callable")) {
                    command.setCallable(XContentMapValues.nodeBooleanValue(m.get("callable")));
                }
                if (m.containsKey("register")) {
                    command.setRegister(XContentMapValues.nodeMapValue(m.get("register"), null));
                }
            } else if (entry instanceof String) {
                command.setSQL((String) entry);
            }
            sql.add(command);
        } catch (IOException e) {
            throw new IllegalArgumentException("SQL command not found", e);
        }
    }
    return sql;
}

private void setContextSettings(String riverResource) throws IOException {
    RiverSettings riverSettings = riverSettings(riverResource);
    Map<String, Object> settings = (Map<String, Object>) riverSettings.settings().get("jdbc");

    riverSettings.settings().put(ColumnRiverFlow.LAST_RUN_TIME, LAST_RUN_TIME);
    riverSettings.settings().put(ColumnRiverFlow.CURRENT_RUN_STARTED_TIME, new TimeValue(new Date().getTime()));

    context.setRiverSettings(riverSettings.settings());
    context.setStatements(SQLCommand.parse(settings));
    context.columnCreatedAt(XContentMapValues.nodeStringValue(settings.get("column_created_at"), null));
    context.columnUpdatedAt(XContentMapValues.nodeStringValue(settings.get("column_updated_at"), null));
    context.columnDeletedAt(XContentMapValues.nodeStringValue(settings.get("column_deleted_at"), null));
    context.columnEscape(true);
    context.setLastRunTimeStampOverlap(getLastRunTimestampOverlap(riverSettings));
}