Boost upload speed

package.json

@@ -74,7 +74,7 @@
     "react-images": "^0.5.19",
     "react-photo-gallery": "^6.1.2",
     "react-transition-group": "^2.2.1",
-    "stream-json": "^1.1.0",
+    "stream-json": "^1.1.4",
     "unzipper": "^0.8.9",
     "uuid": "^3.3.2"
   }

src/components/Menu/MenuContainer.jsx

@@ -257,6 +257,7 @@ export default class MenuContainer extends Component {
             streamArray()
         ]);
         
+
         let localcount = 0;
         let sent = 0;
         let chunk = [];
@@ -267,8 +268,8 @@ export default class MenuContainer extends Component {
             progress: 0
         });
 
+        console.log(`Processing ${file}`);
         console.time("IngestTime");
-
         pipeline
             .on(
                 "data",
@@ -276,7 +277,7 @@ export default class MenuContainer extends Component {
                     chunk.push(data.value);
                     localcount++;
                     
-                    if (localcount % 1 === 0) {
+                    if (localcount % 1000 === 0 || type === "gpomembers" || type === "ous" || type === "domains") {
                         pipeline.pause();
                         await this.uploadData(chunk, type);
                         sent += chunk.length;
@@ -325,6 +326,7 @@ export default class MenuContainer extends Component {
             let arr = data[key].props.chunk()
             let statement = data[key].statement;
             for (let i = 0; i < arr.length; i++){
+                //console.log(arr[i]);
                 await session
                 .run(statement, { props: arr[i] })
                 .catch(function(error) {

src/index.js

@@ -57,15 +57,16 @@ Array.prototype.allEdgesSameType = function() {
 
 Array.prototype.chunk = function() {
     let i = 0;
-    let len = this.length
-    let temp = []
+    let len = this.length;
+    let temp = [];
+    let chunkSize = 10000;
 
-    for (i = 0;i < len; i+= 500){
-        temp.push(this.slice(i, i+500))
+    for (i = 0; i < len; i+= chunkSize){
+        temp.push(this.slice(i, i+chunkSize));
     }
 
     return temp;
-}
+};
 
 if (!Array.prototype.last) {
     Array.prototype.last = function() {