public void start() { apiBootstrap = new ServerBootstrap(); try { // the hub will only have a few connections, so OIO is likely to be faster than NIO in this case! ThreadFactory threadFactory = new NamedThreadFactory("kha-rest-api"); EventLoopGroup commonGroup = new OioEventLoopGroup(0, threadFactory); apiBootstrap.group(commonGroup, commonGroup) .channel(OioServerSocketChannel.class) .localAddress(port) .childOption(ChannelOption.SO_KEEPALIVE, true) .childHandler(channelInitializer); apiBootstrap.bind(); // ChannelFuture f = apiBootstrap.bind().sync(); LOGGER.info("REST API available on http://{}:{}", InetAddress.getLocalHost().getCanonicalHostName(), port); LOGGER.info("WebSockets API available on ws://{}:{}", InetAddress.getLocalHost().getCanonicalHostName(), port); // f.channel().closeFuture().sync(); } catch (Exception e) { LOGGER.error("Can't start API server", e); } }
public static IndexSearcher newSearcher(IndexReader r, boolean wrapWithAssertions) { Random random = new Random(); int threads = 0; final ThreadPoolExecutor ex; if (random.nextBoolean()) { ex = null; } else { threads = randomIntBetween(random, 1, 8); ex = new ThreadPoolExecutor(threads, threads, 0L, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<Runnable>(), new NamedThreadFactory("LuceneTestCase")); } if (ex != null) { r.addReaderClosedListener(new IndexReader.ReaderClosedListener() { @Override public void onClose(IndexReader reader) { shutdownExecutorService(ex); } }); } IndexSearcher ret; if (wrapWithAssertions) { ret = random.nextBoolean() ? new AssertingIndexSearcher(random, r, ex) : new AssertingIndexSearcher(random, r.getContext(), ex); } else { ret = random.nextBoolean() ? new IndexSearcher(r, ex) : new IndexSearcher(r.getContext(), ex); } ret.setSimilarity(new DefaultSimilarity()); return ret; }
public void start() { apiBootstrap = new ServerBootstrap(); ThreadFactory threadFactory = new NamedThreadFactory("kha-webapp"); EventLoopGroup commonGroup = new OioEventLoopGroup(0, threadFactory); try { // the hub will only have a few connections, so OIO is likely to be faster than NIO in this case! apiBootstrap.group(commonGroup, commonGroup) .channel(OioServerSocketChannel.class) .localAddress(port) .childOption(ChannelOption.SO_KEEPALIVE, true) .childHandler(new ChannelInitializer<SocketChannel>() { @Override protected void initChannel(SocketChannel ch) throws Exception { ChannelPipeline pipeline = ch.pipeline(); pipeline.addLast("http-request-decoder", new HttpRequestDecoder()); pipeline.addLast("http-object-aggregator", new HttpObjectAggregator(1048576)); pipeline.addLast("http-response-encoder", new HttpResponseEncoder()); // pipeline.addLast("deflater", new HttpContentDecompressor()); // pipeline.addLast("inflater", new HttpContentCompressor()); pipeline.addLast("chunkedWriter", new ChunkedWriteHandler()); pipeline.addLast("cors", new CorsHandler(corsConfig)); pipeline.addLast("file-handler", new HttpStaticFileServerHandler(hubSiteDirectory, true)); } }); ChannelFuture f = apiBootstrap.bind().sync(); LOGGER.info("WebApp available on http://{}:{}", InetAddress.getLocalHost().getCanonicalHostName(), port); f.channel().closeFuture().sync(); } catch (Exception e) { LOGGER.error("Can't start WebApp server", e); } }
public static IndexSearcher newSearcher(IndexReader r, boolean wrapWithAssertions) { Random random = new Random(); int threads = 0; final ThreadPoolExecutor ex; if (random.nextBoolean()) { ex = null; } else { threads = randomIntBetween(random, 1, 8); ex = new ThreadPoolExecutor(threads, threads, 0L, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<Runnable>(), new NamedThreadFactory("LuceneTestCase")); } if (ex != null) { r.addReaderClosedListener(new IndexReader.ReaderClosedListener() { @Override public void onClose(IndexReader reader) { shutdownExecutorService(ex); } }); } IndexSearcher ret; if (wrapWithAssertions) { ret = random.nextBoolean() ? new AssertingIndexSearcher(random, r, ex) : new AssertingIndexSearcher(random, r.getContext(), ex); } else { ret = random.nextBoolean() ? new IndexSearcher(r, ex) : new IndexSearcher(r.getContext(), ex); } ret.setSimilarity(new ClassicSimilarity()); return ret; }
public OntologyHelper buildOntologyHelper() throws OntologyHelperException { return new OntologyHelperBuilder() .ontologyUri(settings.getOntologyUri()) .labelPropertyUris(convertListToArray(settings.getLabelPropertyUris())) .synonymPropertyUris(convertListToArray(settings.getSynonymPropertyUris())) .definitionPropertyUris(convertListToArray(settings.getDefinitionPropertyUris())) .olsBaseUrl(settings.getOlsBaseUrl()) .ontology(settings.getOlsOntology()) .threadpoolSize(settings.getThreadpoolSize()) .pageSize(settings.getPageSize()) .threadFactory(new NamedThreadFactory("olsOntologyHelper")) .build(); }
public void testConcurrentAccess() throws IOException, InterruptedException { assertEquals(1, searchers.size()); final IndexReader r = DirectoryReader.open(userindex); spellChecker.clearIndex(); assertEquals(2, searchers.size()); addwords(r, spellChecker, "field1"); assertEquals(3, searchers.size()); int num_field1 = this.numdoc(); addwords(r, spellChecker, "field2"); assertEquals(4, searchers.size()); int num_field2 = this.numdoc(); assertEquals(num_field2, num_field1 + 1); int numThreads = 5 + random().nextInt(5); ExecutorService executor = Executors.newFixedThreadPool(numThreads, new NamedThreadFactory("testConcurrentAccess")); SpellCheckWorker[] workers = new SpellCheckWorker[numThreads]; for (int i = 0; i < numThreads; i++) { SpellCheckWorker spellCheckWorker = new SpellCheckWorker(r); executor.execute(spellCheckWorker); workers[i] = spellCheckWorker; } int iterations = 5 + random().nextInt(5); for (int i = 0; i < iterations; i++) { Thread.sleep(100); // concurrently reset the spell index spellChecker.setSpellIndex(this.spellindex); // for debug - prints the internal open searchers // showSearchersOpen(); } spellChecker.close(); executor.shutdown(); // wait for 60 seconds - usually this is very fast but coverage runs could take quite long executor.awaitTermination(60L, TimeUnit.SECONDS); for (int i = 0; i < workers.length; i++) { assertFalse(String.format(Locale.ROOT, "worker thread %d failed", i), workers[i].failed); assertTrue(String.format(Locale.ROOT, "worker thread %d is still running but should be terminated", i), workers[i].terminated); } // 4 searchers more than iterations // 1. at creation // 2. clearIndex() // 2. and 3. during addwords assertEquals(iterations + 4, searchers.size()); assertSearchersClosed(); r.close(); }
public void testHugeN() throws Exception { ExecutorService service = new ThreadPoolExecutor(4, 4, 0L, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<Runnable>(), new NamedThreadFactory("TestIndexSearcher")); IndexSearcher searchers[] = new IndexSearcher[] { new IndexSearcher(reader), new IndexSearcher(reader, service) }; Query queries[] = new Query[] { new MatchAllDocsQuery(), new TermQuery(new Term("field", "1")) }; Sort sorts[] = new Sort[] { null, new Sort(new SortField("field2", SortField.Type.STRING)) }; Filter filters[] = new Filter[] { null, new QueryWrapperFilter(new TermQuery(new Term("field2", "true"))) }; ScoreDoc afters[] = new ScoreDoc[] { null, new FieldDoc(0, 0f, new Object[] { new BytesRef("boo!") }) }; for (IndexSearcher searcher : searchers) { for (ScoreDoc after : afters) { for (Query query : queries) { for (Sort sort : sorts) { for (Filter filter : filters) { searcher.search(query, Integer.MAX_VALUE); searcher.searchAfter(after, query, Integer.MAX_VALUE); searcher.search(query, filter, Integer.MAX_VALUE); searcher.searchAfter(after, query, filter, Integer.MAX_VALUE); if (sort != null) { searcher.search(query, Integer.MAX_VALUE, sort); searcher.search(query, filter, Integer.MAX_VALUE, sort); searcher.search(query, filter, Integer.MAX_VALUE, sort, true, true); searcher.search(query, filter, Integer.MAX_VALUE, sort, true, false); searcher.search(query, filter, Integer.MAX_VALUE, sort, false, true); searcher.search(query, filter, Integer.MAX_VALUE, sort, false, false); searcher.searchAfter(after, query, filter, Integer.MAX_VALUE, sort); searcher.searchAfter(after, query, filter, Integer.MAX_VALUE, sort, true, true); searcher.searchAfter(after, query, filter, Integer.MAX_VALUE, sort, true, false); searcher.searchAfter(after, query, filter, Integer.MAX_VALUE, sort, false, true); searcher.searchAfter(after, query, filter, Integer.MAX_VALUE, sort, false, false); } } } } } } TestUtil.shutdownExecutorService(service); }
public void testDeMorgan() throws Exception { Directory dir1 = newDirectory(); RandomIndexWriter iw1 = new RandomIndexWriter(random(), dir1); Document doc1 = new Document(); doc1.add(newTextField("field", "foo bar", Field.Store.NO)); iw1.addDocument(doc1); IndexReader reader1 = iw1.getReader(); iw1.close(); Directory dir2 = newDirectory(); RandomIndexWriter iw2 = new RandomIndexWriter(random(), dir2); Document doc2 = new Document(); doc2.add(newTextField("field", "foo baz", Field.Store.NO)); iw2.addDocument(doc2); IndexReader reader2 = iw2.getReader(); iw2.close(); BooleanQuery query = new BooleanQuery(); // Query: +foo -ba* query.add(new TermQuery(new Term("field", "foo")), BooleanClause.Occur.MUST); WildcardQuery wildcardQuery = new WildcardQuery(new Term("field", "ba*")); wildcardQuery.setRewriteMethod(MultiTermQuery.SCORING_BOOLEAN_QUERY_REWRITE); query.add(wildcardQuery, BooleanClause.Occur.MUST_NOT); MultiReader multireader = new MultiReader(reader1, reader2); IndexSearcher searcher = newSearcher(multireader); assertEquals(0, searcher.search(query, 10).totalHits); final ExecutorService es = Executors.newCachedThreadPool(new NamedThreadFactory("NRT search threads")); searcher = new IndexSearcher(multireader, es); if (VERBOSE) System.out.println("rewritten form: " + searcher.rewrite(query)); assertEquals(0, searcher.search(query, 10).totalHits); es.shutdown(); es.awaitTermination(1, TimeUnit.SECONDS); multireader.close(); reader1.close(); reader2.close(); dir1.close(); dir2.close(); }
public void testDeMorgan() throws Exception { Directory dir1 = newDirectory(); RandomIndexWriter iw1 = new RandomIndexWriter(random(), dir1); Document doc1 = new Document(); doc1.add(newTextField("field", "foo bar", Field.Store.NO)); iw1.addDocument(doc1); IndexReader reader1 = iw1.getReader(); iw1.close(); Directory dir2 = newDirectory(); RandomIndexWriter iw2 = new RandomIndexWriter(random(), dir2); Document doc2 = new Document(); doc2.add(newTextField("field", "foo baz", Field.Store.NO)); iw2.addDocument(doc2); IndexReader reader2 = iw2.getReader(); iw2.close(); BooleanQuery query = new BooleanQuery(); // Query: +foo -ba* query.add(new TermQuery(new Term("field", "foo")), BooleanClause.Occur.MUST); WildcardQuery wildcardQuery = new WildcardQuery(new Term("field", "ba*")); wildcardQuery.setRewriteMethod(MultiTermQuery.SCORING_BOOLEAN_QUERY_REWRITE); query.add(wildcardQuery, BooleanClause.Occur.MUST_NOT); MultiReader multireader = new MultiReader(reader1, reader2); IndexSearcher searcher = new IndexSearcher(multireader); assertEquals(0, searcher.search(query, 10).totalHits); final ExecutorService es = Executors.newCachedThreadPool(new NamedThreadFactory("NRT search threads")); searcher = new IndexSearcher(multireader, es); if (VERBOSE) System.out.println("rewritten form: " + searcher.rewrite(query)); assertEquals(0, searcher.search(query, 10).totalHits); es.shutdown(); es.awaitTermination(1, TimeUnit.SECONDS); multireader.close(); reader1.close(); reader2.close(); dir1.close(); dir2.close(); }
public void testHugeN() throws Exception { ExecutorService service = new ThreadPoolExecutor(4, 4, 0L, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<Runnable>(), new NamedThreadFactory("TestIndexSearcher")); IndexSearcher searchers[] = new IndexSearcher[] { new IndexSearcher(reader), new IndexSearcher(reader, service) }; Query queries[] = new Query[] { new MatchAllDocsQuery(), new TermQuery(new Term("field", "1")) }; Sort sorts[] = new Sort[] { null, new Sort(new SortField("field2", SortField.Type.STRING)) }; Filter filters[] = new Filter[] { null, new QueryWrapperFilter(new TermQuery(new Term("field2", "true"))) }; ScoreDoc afters[] = new ScoreDoc[] { null, new FieldDoc(0, 0f, new Object[] { new BytesRef("boo!") }) }; for (IndexSearcher searcher : searchers) { for (ScoreDoc after : afters) { for (Query query : queries) { for (Sort sort : sorts) { for (Filter filter : filters) { searcher.search(query, Integer.MAX_VALUE); searcher.searchAfter(after, query, Integer.MAX_VALUE); searcher.search(query, filter, Integer.MAX_VALUE); searcher.searchAfter(after, query, filter, Integer.MAX_VALUE); if (sort != null) { searcher.search(query, Integer.MAX_VALUE, sort); searcher.search(query, filter, Integer.MAX_VALUE, sort); searcher.search(query, filter, Integer.MAX_VALUE, sort, true, true); searcher.search(query, filter, Integer.MAX_VALUE, sort, true, false); searcher.search(query, filter, Integer.MAX_VALUE, sort, false, true); searcher.search(query, filter, Integer.MAX_VALUE, sort, false, false); searcher.searchAfter(after, query, filter, Integer.MAX_VALUE, sort); searcher.searchAfter(after, query, filter, Integer.MAX_VALUE, sort, true, true); searcher.searchAfter(after, query, filter, Integer.MAX_VALUE, sort, true, false); searcher.searchAfter(after, query, filter, Integer.MAX_VALUE, sort, false, true); searcher.searchAfter(after, query, filter, Integer.MAX_VALUE, sort, false, false); } } } } } } _TestUtil.shutdownExecutorService(service); }