/** * Searches the lucene store for a specific query * * @param <T> What type of information are we searching * @param clazz The class of the information we are searching * @param queryText The query text * @return list of entities * @throws ParseException the parse exception */ public final <T extends BaseEntity> List<Object[]> search(final Class<T> clazz, final String queryText) throws ParseException { final FullTextEntityManager fullTextEntityManager = Search.getFullTextEntityManager(entityManager); final SearchFactory searchFactory = fullTextEntityManager.getSearchFactory(); final QueryParser parser = new MultiFieldQueryParser(getClassLuceneFields(clazz), searchFactory.getAnalyzer(clazz)); final List<Query> parsedQueries = Arrays.stream(queryText.split("AND")) .map(e -> parseQuery(e, parser)) .filter(Objects::nonNull) .collect(Collectors.toList()); final BooleanQuery.Builder bq = new BooleanQuery.Builder(); parsedQueries.forEach(e -> bq.add(e, BooleanClause.Occur.MUST)); final FullTextQuery jpaQuery = fullTextEntityManager.createFullTextQuery(bq.build(), clazz); jpaQuery.setProjection(ProjectionConstants.SCORE, ProjectionConstants.EXPLANATION, ProjectionConstants.THIS); return (List<Object[]>) jpaQuery.getResultList(); }
@Override public List<Long> getIdResults(String terminology, String version, String branch, String query, String literalField, Class<?> clazz, PfsParameter pfs, int[] totalCt, EntityManager manager) throws Exception { final FullTextQuery fullTextQuery = helper(terminology, version, branch, query, literalField, clazz, pfs, manager); totalCt[0] = fullTextQuery.getResultSize(); // Perform the final query and save score values fullTextQuery.setProjection(ProjectionConstants.ID); final List<Long> ids = new ArrayList<>(); @SuppressWarnings("unchecked") final List<Object[]> results = fullTextQuery.getResultList(); for (final Object[] result : results) { Long l = (Long) result[0]; ids.add(l); } return ids; }
@Override public List<Long> getIdResults(String terminology, String version, String branch, String query, String literalField, Class<?> clazz, PfsParameter pfs, int[] totalCt, EntityManager manager) throws Exception { final FullTextQuery fullTextQuery = helper(terminology, version, branch, query, literalField, clazz, pfs, totalCt, manager); // Retrieve the scores for the returned objects fullTextQuery.setProjection(ProjectionConstants.ID); final List<Long> ids = new ArrayList<>(); @SuppressWarnings("unchecked") final List<Object[]> results = fullTextQuery.getResultList(); for (final Object[] result : results) { final Long id = (Long) result[0]; ids.add(id); } return ids; }
@Nullable @SuppressWarnings({ CompilerWarnings.UNCHECKED }) private Long[] processKeywords(EntityManager entityManager, @Nullable Integer maxResults) { FullTextEntityManager fullTextEntityManager = Search.getFullTextEntityManager(entityManager); QueryBuilder queryBuilder = fullTextEntityManager.getSearchFactory().buildQueryBuilder().forEntity(this.entityImplClass).get(); BooleanJunction<?> conjunction = queryBuilder.bool(); PropertyMetadata keywordPropMetadata; for (String keywordPropName : this.keywords.keySet()) { conjunction = conjunction.must(queryBuilder.keyword() .onFields((keywordPropMetadata = this.entityPropMetadatas.get(keywordPropName)).getLowercaseFieldName(), keywordPropMetadata.getEdgeNgramFieldName(), keywordPropMetadata.getNgramFieldName(), keywordPropMetadata.getPhoneticFieldName()) .matching(StringUtils.join(this.keywords.get(keywordPropName), StringUtils.SPACE)).createQuery()); } FullTextQuery fullTextQuery = fullTextEntityManager.createFullTextQuery(new InterceptingQuery(conjunction.createQuery()), this.entityImplClass); fullTextQuery.setProjection(ProjectionConstants.ID); if (maxResults != null) { fullTextQuery.setMaxResults(maxResults); } return ((List<Object[]>) fullTextQuery.getResultList()).stream().map(fullTextResult -> ((Long) fullTextResult[0])).toArray(Long[]::new); }
@Override public void purge(Class<?> entityClass, Query query, TransactionContext tc) { HSearchQuery hsQuery = this.createQuery( query, entityClass ); int count = hsQuery.queryResultSize(); int processed = 0; while ( processed < count ) { hsQuery.firstResult( processed ).maxResults( 10 ); processed += 10; for ( Object[] vals : hsQuery.queryProjection( ProjectionConstants.OBJECT_CLASS, ProjectionConstants.ID ) ) { this.purge( entityClass, (Serializable) vals[1], tc ); } } }
@Override public <T extends HasId> List<T> getQueryResults(String terminology, String version, String branch, String query, String literalField, Class<T> clazz, PfsParameter pfs, int[] totalCt, EntityManager manager) throws Exception { final FullTextQuery fullTextQuery = helper(terminology, version, branch, query, literalField, clazz, pfs, manager); totalCt[0] = fullTextQuery.getResultSize(); // Perform the final query and save score values fullTextQuery.setProjection(ProjectionConstants.SCORE, ProjectionConstants.THIS); final List<T> classes = new ArrayList<>(); @SuppressWarnings("unchecked") final List<Object[]> results = fullTextQuery.getResultList(); for (final Object[] result : results) { Object score = result[0]; @SuppressWarnings("unchecked") T t = (T) result[1]; // TODO is this needed? or is it dangerous if (t == null) { continue; } classes.add(t); // normalize results to a "good match" (lucene score of 5.0+) // Double normScore = Math.log(Math.max(5, scoreMap.get(sr.getId())) / // Math.log(5)); // cap the score to a maximum of 5.0 and normalize to the range [0,1] Float normScore = Math.min(5, Float.valueOf(score.toString())) / 5; // store the score scoreMap.put(t.getId(), normScore.floatValue()); } return classes; }
@Override public <T extends HasId> List<T> getQueryResults(String terminology, String version, String branch, String query, String literalField, Class<T> clazz, PfsParameter pfs, int[] totalCt, EntityManager manager) throws Exception { final FullTextQuery fullTextQuery = helper(terminology, version, branch, query, literalField, clazz, pfs, totalCt, manager); // Retrieve the scores for the returned objects fullTextQuery.setProjection(ProjectionConstants.SCORE, ProjectionConstants.THIS); final List<T> classes = new ArrayList<>(); @SuppressWarnings("unchecked") final List<Object[]> results = fullTextQuery.getResultList(); String literalQuery = null; if (query != null && query.startsWith("\"") && query.endsWith("\"")) { literalQuery = query.substring(1, query.length() - 1); } for (final Object[] result : results) { final Object score = result[0]; @SuppressWarnings("unchecked") final T t = (T) result[1]; classes.add(t); // cap the score to a maximum of 1.0 Float normScore = Math.min(1, Float.valueOf(score.toString())); // bump up relevance of exact match on terminology id if (literalQuery != null && literalQuery.equals(((AbstractComponent) t).getTerminologyId())) { normScore = 1.0f; } // store the score for later retrieval scoreMap.put(t.getId(), normScore.floatValue()); } return classes; }
@SuppressWarnings({"rawtypes", "unchecked"}) @Override public List query(EntityProvider entityProvider, Fetch fetchType) { List<Object> ret; List<Object[]> projected = this.queryProjection( ProjectionConstants.OBJECT_CLASS, ProjectionConstants.ID ); if ( fetchType == Fetch.FIND_BY_ID ) { ret = projected.stream().map( (arr) -> { if ( arr[1] == null ) { LOGGER.info( "null id in index ommited for query" ); return null; } Object obj = entityProvider.get( (Class<?>) arr[0], arr[1], this.hints ); if ( obj == null ) { LOGGER.info( "ommiting object of class " + arr[0] + " and id " + arr[1] + " which was found in the index but not in the database!" ); } return obj; } ).filter( (obj) -> obj != null ).collect( Collectors.toList() ); } else { ret = new ArrayList<>( projected.size() ); Map<Class<?>, List<Object>> idsForClass = new HashMap<>(); List<Object[]> originalOrder = new ArrayList<>(); Map<Class<?>, Map<Object, Object>> classToIdToObject = new HashMap<>(); // split the ids for each class (and also make sure the original // order is saved. this is needed even for only one class) projected.stream().forEach( (arr) -> { if ( arr[1] == null ) { LOGGER.info( "null id in index ommited for query" ); return; } originalOrder.add( arr ); idsForClass.computeIfAbsent( (Class<?>) arr[0], (clazz) -> new ArrayList<>() ).add( arr[1] ); // just make sure the map is already created, // we do this here to not clutter the following code classToIdToObject.computeIfAbsent( (Class<?>) arr[0], (clz) -> new HashMap<>() ); } ); // get all entities of the same type in one batch idsForClass.entrySet().forEach( (entry) -> entityProvider.getBatch( entry.getKey(), entry.getValue(), this.hints ).stream().forEach( (object) -> { Class<?> entityClass = entry.getKey(); Object id = this.searchIntegrator.getIndexBinding( entityClass ) .getDocumentBuilder() .getId( object ); classToIdToObject.get( entityClass ).put( id, object ); } ) ); // and put everything back into order originalOrder.stream().forEach( (arr) -> { Object value = classToIdToObject.get( arr[0] ).get( arr[1] ); if ( value == null ) { LOGGER.info( "ommiting object of class " + arr[0] + " and id " + arr[1] + " which was found in the index but not in the database!" ); } else { ret.add( classToIdToObject.get( arr[0] ).get( arr[1] ) ); } } ); } if ( ret.size() != projected.size() ) { LOGGER.info( "returned size was not equal to projected size" ); } return ret; }
@Override public void delete( Class<?> entityClass, List<Class<?>> inIndexOf, Object id, EntityProvider entityProvider, Transaction tx) { for ( Class<?> indexClass : inIndexOf ) { RehashedTypeMetadata metadata = IndexUpdater.this.metadataForIndexRoot.get( indexClass ); List<String> fields = metadata.getIdFieldNamesForType().get( entityClass ); for ( String field : fields ) { DocumentFieldMetadata metaDataForIdField = metadata.getDocumentFieldMetadataForIdFieldName().get( field ); SingularTermDeletionQuery.Type idType = metadata.getSingularTermDeletionQueryTypeForIdFieldName() .get( entityClass ); Object idValueForDeletion; if ( idType == SingularTermDeletionQuery.Type.STRING ) { FieldBridge fb = metaDataForIdField.getFieldBridge(); if ( !(fb instanceof StringBridge) ) { throw new IllegalArgumentException( "no TwoWayStringBridge found for field: " + field ); } idValueForDeletion = ((StringBridge) fb).objectToString( id ); } else { idValueForDeletion = id; } if ( indexClass.equals( entityClass ) ) { this.searchIntegrator.getWorker().performWork( new Work( entityClass, (Serializable) id, WorkType.DELETE ), tx ); } else { HSQuery hsQuery = this.searchIntegrator .createHSQuery() .targetedEntities( Collections.singletonList( indexClass ) ) .luceneQuery( this.searchIntegrator.buildQueryBuilder() .forEntity( indexClass ) .get() .keyword() .onField( field ) .matching( idValueForDeletion ) .createQuery() ); int count = hsQuery.queryResultSize(); int processed = 0; // this was just contained somewhere // so we have to update the containing entity while ( processed < count ) { for ( EntityInfo entityInfo : hsQuery.firstResult( processed ).projection( ProjectionConstants.ID ).maxResults( HSQUERY_BATCH ) .queryEntityInfos() ) { Serializable originalId = (Serializable) entityInfo.getProjection()[0]; Object original = entityProvider.get( indexClass, originalId ); if ( original != null ) { this.update( original, tx ); } else { // original is not available in the // database, but it will be deleted by its // own delete event // TODO: log this? } } processed += HSQUERY_BATCH; } } } } }
