private DBCursor createCursor() { DBCollection oplog = _mongo.getDB("local").getCollection("oplog.rs"); BSONTimestamp startingTimestamp = getStartingTimestamp(); DBCursor cursor; if (startingTimestamp == null) { log.info("Tailing the oplog from the beginning..."); cursor = oplog.find(); } else { log.info("Tailing the oplog from " + startingTimestamp); BasicDBObject query = new BasicDBObject("ts", new BasicDBObject("$gt", startingTimestamp)); cursor = oplog.find(query); cursor.addOption(Bytes.QUERYOPTION_OPLOGREPLAY); } cursor.addOption(Bytes.QUERYOPTION_NOTIMEOUT); cursor.addOption(Bytes.QUERYOPTION_TAILABLE); cursor.addOption(Bytes.QUERYOPTION_AWAITDATA); return cursor; }
MediaItemsVisualIndexer(String host, String dbname, String collectionName, String webServiceHost, String indexCollection, String[] codebookFiles, String pcaFile) throws Exception { this.mongo = new MongoClient(host); this.db = mongo.getDB(dbname); this.input_collection = db.getCollection(collectionName); input_collection.addOption(Bytes.QUERYOPTION_NOTIMEOUT); ImageVectorization.setFeatureExtractor(new SURFExtractor()); VladAggregatorMultipleVocabularies vladAggregator = new VladAggregatorMultipleVocabularies(codebookFiles, numCentroids, AbstractFeatureExtractor.SURFLength); ImageVectorization.setVladAggregator(vladAggregator); int initialLength = numCentroids.length * numCentroids[0] * AbstractFeatureExtractor.SURFLength; PCA pca = new PCA(targetLengthMax, 1, initialLength, true); pca.loadPCAFromFile(pcaFile); ImageVectorization.setPcaProjector(pca); this.visualIndexHandler = new VisualIndexHandler(webServiceHost, indexCollection); }
public RecordIterator getRecords() { verifyProperty(dbname, "database"); verifyProperty(collectionName, "collection"); try { final MongoClient mongo; final DB database; final DBCollection collection; final DBCursor result; final DBObject queryDocument; final DBObject projectionDocument; // authentication mecanism via MONGODB-CR authentication http://docs.mongodb.org/manual/core/authentication/#authentication-mongodb-cr if(auth.equals(AUTH_ON_DB)){ verifyProperty(username, "user-name"); verifyProperty(password, "password"); mongo = new MongoClient( new ServerAddress(mongouri, port), Arrays.asList(MongoCredential.createMongoCRCredential(username, dbname, password.toCharArray())) ); } else if(auth.equals(AUTH_ON_ADMIN)){ verifyProperty(username, "user-name"); verifyProperty(password, "password"); mongo = new MongoClient( new ServerAddress(mongouri, port), Arrays.asList(MongoCredential.createMongoCRCredential(username, AUTH_ON_ADMIN, password.toCharArray())) ); } else{ mongo = new MongoClient(new ServerAddress(mongouri, port)); } // get db, collection database = mongo.getDB(dbname); collection = database.getCollection(collectionName); // execute query queryDocument = (DBObject)JSON.parse(query); if(projection==null){ result = collection.find(queryDocument); } else{ projectionDocument = (DBObject)JSON.parse(projection); result = collection.find(queryDocument, projectionDocument); } // See: http://api.mongodb.org/java/current/com/mongodb/DBCursor.html#addOption(int) // and http://api.mongodb.org/java/current/com/mongodb/Bytes.html#QUERYOPTION_NOTIMEOUT if(noTimeOut){ result.addOption(Bytes.QUERYOPTION_NOTIMEOUT); } return new MongoDBIterator(result, mongo); } catch (UnknownHostException e) { throw new RuntimeException(e); } catch (Exception ex){ throw new DukeException(ex); } }
protected void initQuery(MongoConnection conn) throws IOException { if (query != null) cur = conn.coll.find((DBObject) JSON.parse(query)); else cur = conn.coll.find(); cur.addOption(Bytes.QUERYOPTION_NOTIMEOUT).batchSize(1000); }
@Override public Iterator<String> getAllChunkIds(long maxLastModifiedTime) throws Exception { DBCollection collection = getBlobCollection(); DBObject fields = new BasicDBObject(); fields.put(MongoBlob.KEY_ID, 1); QueryBuilder builder = new QueryBuilder(); if (maxLastModifiedTime != 0 && maxLastModifiedTime != -1) { builder.and(MongoBlob.KEY_LAST_MOD).lessThanEquals(maxLastModifiedTime); } final DBCursor cur = collection.find(builder.get(), fields).hint(fields) .addOption(Bytes.QUERYOPTION_SLAVEOK); //TODO The cursor needs to be closed return new AbstractIterator<String>() { @Override protected String computeNext() { if (cur.hasNext()) { MongoBlob blob = (MongoBlob) cur.next(); if (blob != null) { return blob.getId(); } } return endOfData(); } }; }
@SuppressWarnings("unchecked") static Object fetchFieldDocument( Object fieldValue, byte fieldNativeDataType ) { if( fieldNativeDataType == BSON.UNDEFINED ) fieldNativeDataType = Bytes.getType( fieldValue ); if( fieldNativeDataType == BSON.ARRAY ) { if( ! (fieldValue instanceof List) ) return null; // fetch nested document, if exists, for each element in array List<Document> documentList = new ArrayList<Document>( ); for( Object valueInList : (List<?>)fieldValue ) { Object listElementObj = fetchFieldDocument( valueInList ); if( listElementObj == null ) // at least one element in array is not a nested doc return null; if( listElementObj instanceof List ) documentList.addAll( (List<Document>)listElementObj ); // collapse into the same list else documentList.add( (Document)listElementObj ); } return documentList; // return nested documents in an array } Document fieldObjValue = null; if( fieldValue instanceof Document ) { fieldObjValue = (Document) fieldValue; }else { // log and ignore getLogger().log( Level.INFO, "Ignoring error in fetching of unsupported BSON.OBJECT type"+ fieldValue ); //$NON-NLS-1$ } return fieldObjValue; }
/** * * @param obj the object to be serialized * @param buf StringBuilder containing the JSON representation of the object */ @Override public void serialize(Object obj, StringBuilder buf){ obj = Bytes.applyEncodingHooks( obj ); if(obj == null) { buf.append(" null "); return; } ObjectSerializer serializer = null; List<Class<?>> ancestors; ancestors = ClassMap.getAncestry(obj.getClass()); for (final Class<?> ancestor : ancestors) { serializer = _serializers.get(ancestor); if (serializer != null) break; } if (serializer == null && obj.getClass().isArray()) serializer = _serializers.get(Object[].class); if (serializer == null) throw new RuntimeException( "json can't serialize type : " + obj.getClass() ); serializer.serialize(obj, buf); }
@Override public void serialize(Object obj, StringBuilder buf) { DBObject externalForm = new BasicDBObject(); externalForm.put("$regex", obj.toString()); if (((Pattern) obj).flags() != 0) externalForm.put("$options", Bytes.regexFlags(((Pattern) obj).flags())); serializer.serialize(externalForm, buf); }
@Test public void query() { // 查询所有 // queryAll(); // 查询id = 4de73f7acd812d61b4626a77 print("find id = 4de73f7acd812d61b4626a77: " + users.find(new BasicDBObject("_id", new ObjectId("4de73f7acd812d61b4626a77"))).toArray()); // 查询age = 24 print("find age = 24: " + users.find(new BasicDBObject("age", 24)).toArray()); // 查询age >= 24 print("find age >= 24: " + users.find(new BasicDBObject("age", new BasicDBObject("$gte", 24))).toArray()); print("find age <= 24: " + users.find(new BasicDBObject("age", new BasicDBObject("$lte", 24))).toArray()); print("查询age!=25:" + users.find(new BasicDBObject("age", new BasicDBObject("$ne", 25))).toArray()); print("查询age in 25/26/27:" + users.find(new BasicDBObject("age", new BasicDBObject(QueryOperators.IN, new int[] { 25, 26, 27 }))).toArray()); print("查询age not in 25/26/27:" + users.find(new BasicDBObject("age", new BasicDBObject(QueryOperators.NIN, new int[] { 25, 26, 27 }))).toArray()); print("查询age exists 排序:" + users.find(new BasicDBObject("age", new BasicDBObject(QueryOperators.EXISTS, true))).toArray()); print("只查询age属性:" + users.find(null, new BasicDBObject("age", true)).toArray()); print("只查属性:" + users.find(null, new BasicDBObject("age", true), 0, 2).toArray()); print("只查属性:" + users.find(null, new BasicDBObject("age", true), 0, 2, Bytes.QUERYOPTION_NOTIMEOUT).toArray()); // 只查询一条数据,多条去第一条 print("findOne: " + users.findOne()); print("findOne: " + users.findOne(new BasicDBObject("age", 26))); print("findOne: " + users.findOne(new BasicDBObject("age", 26), new BasicDBObject("name", true))); // 查询修改、删除 print("findAndRemove 查询age=25的数据,并且删除: " + users.findAndRemove(new BasicDBObject("age", 25))); // 查询age=26的数据,并且修改name的值为Abc print("findAndModify: " + users.findAndModify(new BasicDBObject("age", 26), new BasicDBObject("name", "Abc"))); print("findAndModify: " + users.findAndModify(new BasicDBObject("age", 28), // 查询age=28的数据 new BasicDBObject("name", true), // 查询name属性 new BasicDBObject("age", true), // 按照age排序 false, // 是否删除,true表示删除 new BasicDBObject("name", "Abc"), // 修改的值,将name修改成Abc true, true)); queryAll(); }
private void addDataType( Object fieldValue ) { // add the data type of given fieldValue to existing set, if exists; // the same named field set in a different doc may have a different data type byte nativeBSonDataTypeCode = Bytes.getType( fieldValue ); if( m_nativeDataTypes == null ) m_nativeDataTypes = new HashSet<Integer>(2); if ( nativeBSonDataTypeCode == -1 ) { if ( fieldValue instanceof Document ) { nativeBSonDataTypeCode = Bytes.OBJECT; } } m_nativeDataTypes.add( Integer.valueOf( nativeBSonDataTypeCode ) ); // check if field value contains a document, // iteratively get field document's nested metadata if( nativeBSonDataTypeCode == BSON.ARRAY ) { if( fieldValue instanceof java.util.List ) { java.util.List<?> listOfObjects = (java.util.List<?>)fieldValue; if( listOfObjects.size() > 0 ) { // use first element in array to determine metadata addDataType( listOfObjects.get(0) ); // handles nested arrays return; } } } Object fieldObjValue = ResultDataHandler.fetchFieldDocument( fieldValue, nativeBSonDataTypeCode ); if( fieldObjValue != null ) // contains nested document { if( m_childDocMetaData == null ) m_childDocMetaData = sm_factory.new DocumentsMetaData(); m_childDocMetaData.addDocumentMetaData( fieldObjValue, this ); } }