diff --git a/package.json b/package.json index 6413951..7eb08f4 100644 --- a/package.json +++ b/package.json @@ -74,7 +74,7 @@ "react-images": "^0.5.19", "react-photo-gallery": "^6.1.2", "react-transition-group": "^2.2.1", - "stream-json": "^1.1.0", + "stream-json": "^1.1.4", "unzipper": "^0.8.9", "uuid": "^3.3.2" } diff --git a/src/components/Menu/MenuContainer.jsx b/src/components/Menu/MenuContainer.jsx index 91b5dd7..fee76cd 100644 --- a/src/components/Menu/MenuContainer.jsx +++ b/src/components/Menu/MenuContainer.jsx @@ -256,6 +256,7 @@ export default class MenuContainer extends Component { withParser({ filter: type }), streamArray() ]); + let localcount = 0; let sent = 0; @@ -267,16 +268,16 @@ export default class MenuContainer extends Component { progress: 0 }); + console.log(`Processing ${file}`); console.time("IngestTime"); - pipeline .on( "data", async function(data) { chunk.push(data.value); localcount++; - - if (localcount % 1 === 0) { + + if (localcount % 1000 === 0 || type === "gpomembers" || type === "ous" || type === "domains") { pipeline.pause(); await this.uploadData(chunk, type); sent += chunk.length; @@ -325,6 +326,7 @@ export default class MenuContainer extends Component { let arr = data[key].props.chunk() let statement = data[key].statement; for (let i = 0; i < arr.length; i++){ + //console.log(arr[i]); await session .run(statement, { props: arr[i] }) .catch(function(error) { diff --git a/src/index.js b/src/index.js index 694ff7d..8b24d21 100644 --- a/src/index.js +++ b/src/index.js @@ -57,15 +57,16 @@ Array.prototype.allEdgesSameType = function() { Array.prototype.chunk = function() { let i = 0; - let len = this.length - let temp = [] + let len = this.length; + let temp = []; + let chunkSize = 10000; - for (i = 0;i < len; i+= 500){ - temp.push(this.slice(i, i+500)) + for (i = 0; i < len; i+= chunkSize){ + temp.push(this.slice(i, i+chunkSize)); } return temp; -} +}; if (!Array.prototype.last) { Array.prototype.last = function() {