0

So I have about 1M records to first insert and then subsequently update an array in them whenever I run the script.

A typical record

    { 

    "_id" : Object(), 

    "id" : "my_unique_id", 

    "name" : "my name", 

    "books" : [
{
"name" : "The last song",
"author" : "some_author"
}
] 

    }

So I have about 1M of these records when the database has no documents, It can simply insert the data, but when the database has the records the database should only update "books" array and insert the new data we have, I mean push into the books.

Currently, I am doing this, looking for documents, if exists I push into it and if not simply inserting it, but as my database went upto 25k records, I am getting this error.

Code for inserting documents

async function insert_document(my_object, collection_name) {

insertion_requests_values++

while (insertion_requests_values > insertion_requests_values_threashold)
{
    await sleep(300)
}


let stats = my_object["statistics"]
stats["time"] = new Date().toISOString().slice(0, 19)
    .replace('T', ' ')+'.'+ new Date().getMilliseconds();



my_object["statistics"] = []
my_object["statistics"].push(stats)





// The code is commented becase it simply insets a document, we need updations

// database_object.collection(collection_name).insertOne(my_object, function(err, res) {
//     if (err) {
//
//         errorLog(err)
//         return
//     }
//     log("documents inserted = " + ++inserted_documents);
// });




var query = { id: my_object.id};
database_object.collection(collection_name).find(query).toArray(function(err, result) {
    insertion_requests_values--

    if (err) {


        errorLog(err)
        return
    }
    if(result.length > 0 ) {
        let newValue = result[0]

        try {


            newValue["statistics"].push(stats)
        }
        catch (e) {
            newValue["statistics"] = []
            newValue["statistics"].push(stats)

        }
        database_object.collection(collection_name).updateOne(query, { $set: newValue },  { upsert: true }, function(err, res) {
            // insertion_requests_values--
            if (err) {

                errorLog(err)
                return
            }
            log("document updated= " + ++updated_documents);
        });


        //The code is commented becuase it deletes and add new document
        //The code in the comments is basically looking for existing documents and then deleting
        // that document and inserting new one, same as updating stuff
        // database_object.collection(collection_name).deleteOne(query, function(err, obj) {
        //     if (err) {
        //
        //         errorLog(err)
        //         return
        //     }

        // database_object.collection(collection_name).insertOne(my_object, function(err, res) {
        //     if (err) {
        //
        //         errorLog(err)
        //         return
        //     }
        //     log("document updated= " + ++updated_documents);
        // });
        // });


    }
    else {


        database_object.collection(collection_name).insertOne(my_object, function(err, res) {
            // insertion_requests_values--

            if (err) {

                errorLog(err)
                return
            }
            log("documents inserted = " + ++inserted_documents);
        });

    }
});

}

Error mongo is giving...

2019-08-18T16:02:14.823+0000 I COMMAND [conn16] command TubeC.TubeCVideos command: find { find: "TubeCVideos", filter: { id: "nHpdrzJA5NY" }, returnKey: false, showRecordId: false, $db: "TubeC" } planSummary: COLLSCAN keysExamined:0 docsExamined:28108 cursorExhausted:1 numYields:221 nreturned:1 reslen:3753 locks:{ Global: { acquireCount: { r: 222 } }, Database: { acquireCount: { r: 222 } }, Collection: { acquireCount: { r: 222 } } } storage:{} protocol:op_msg 129ms

How can I achieve that with 1M records using NodeJs?

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Browse other questions tagged or ask your own question.