sundowndev
/
hetical-forensic
mirror of https://github.com/sundowndev/hetical-forensic.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Pull command

master
sundowndev 2019-02-18 23:20:03 +01:00
parent dfc83dd700
commit 98eb29e753
1 changed files with 58 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

83
commands/pull/index.js
View File

@ -4,7 +4,7 @@ export const description = 'Pull data from outils.hetic.net';
const rp = require("request-promise");
const cheerio = require('cheerio');
export const action = async ({ req, logger }) => {
export const action = async ({ logger }) => {
const options = {
followAllRedirects: true,
method: 'GET',
@ -12,7 +12,7 @@ export const action = async ({ req, logger }) => {
headers: {
'cache-control': 'no-cache',
'User-Agent': 'User-Agent: Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:65.0) Gecko/20100101 Firefox/65.0',
'Cookie': `PHPSESSID=${global.user.phpsessid};`,
'Cookie': `${global.user.phpsessid};`,
},
transform: function (body) {
return cheerio.load(body);
@ -23,7 +23,6 @@ export const action = async ({ req, logger }) => {
await rp(options)
.then(function ($) {
// logger.log(options.headers);
logger.info('Successfully fetched outils.hetic.net');
$('select#select_list_promo_students option').each(function (i, elem) {
@ -43,7 +42,7 @@ export const action = async ({ req, logger }) => {
headers: {
'cache-control': 'no-cache',
'User-Agent': 'User-Agent: Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:65.0) Gecko/20100101 Firefox/65.0',
'Cookie': `PHPSESSID=${global.user.phpsessid};`,
'Cookie': `${global.user.phpsessid};`,
},
transform: function (body) {
return cheerio.load(body);
@ -52,41 +51,76 @@ export const action = async ({ req, logger }) => {
await rp(options)
.then(async ($) => {
// logger.debug($('body'));
logger.info(`Fetching students from ${promotion} promotion...`);
await $('table.liste_insertions.liste_insertions_stage tbody tr').each(function (i, student) {
$('table.liste_insertions.liste_insertions_stage tbody tr').each(async function () {
$('td a', this).each(function (index, col) {
if ($(col).text().slice(-1) === ' ') $(col).text($(col).text().slice(0, -1));
else if ($(col).text() === ' ') $(col).text('');
});
let uid = $('td:nth-child(1) a', this).attr('href').replace('/ogi/profil/', '');
let lastName = $('td:nth-child(1)', this).text();
let firstName = $('td:nth-child(2)', this).text();
let heticEmail = `${firstName.toLowerCase()}.${lastName.toLowerCase()}@hetic.net`
const student = {};
student.uid = $('td:nth-child(1) a', this).attr('href').replace('/ogi/profil/', '');
student.lastName = $('td:nth-child(1)', this).text();
student.firstName = $('td:nth-child(2)', this).text();
student.heticEmail = `${student.firstName.toLowerCase()}.${student.lastName.toLowerCase()}@hetic.net`
.normalize('NFD')
.replace(/[\u0300-\u036f]/g, "")
.replace(new RegExp(' ', 'g'), '');
student.email = null;
student.birthdate = null
student.phone = null;
student.cursus = null;
student.picture = `https://outils.hetic.net/external/picture/${student.uid}`;
student.enterprise = $('td:nth-child(3)', this).text() || null;
student.country = $('td:nth-child(4)', this).text() || 'France';
student.promotion = promotion;
await rp({
followAllRedirects: true,
method: 'GET',
url: 'https://outils.hetic.net/ogi/profil/' + student.uid,
headers: {
'cache-control': 'no-cache',
'User-Agent': 'User-Agent: Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:65.0) Gecko/20100101 Firefox/65.0',
'Cookie': `${global.user.phpsessid};`,
},
transform: function (body) {
return cheerio.load(body);
},
})
.then(function ($) {
let secondLi = $('.left .item:nth-child(1) ul li:nth-child(2)').text();
student.email = $('.left .item:nth-child(1) ul li:nth-child(1)').text() || null;
student.cursus = $('.right.right_first .infos h3').text().replace(` P${student.promotion}`, '') || null;
if (secondLi.match(/^(?:n\é\(e\)\ le\ )(?:.*)/)) {
secondLi = secondLi.replace('né(e) le ', '') || null;
secondLi = secondLi.split('/');
student.birthdate = new Date(`${secondLi[2]}-${secondLi[1]}-${secondLi[0]}`);
} else if (secondLi.match(/^[0-9]{8,}/)) {
student.phone = secondLi || null;
}
})
let studentObj = {
uid: uid,
firstName: firstName,
lastName: lastName,
heticEmail: heticEmail,
email: null,
phone: null,
cursus: null,
birthdate: null,
picture: `https://outils.hetic.net/external/picture/${uid}`,
promotion: promotion,
enterprise: $('td:nth-child(3)', this).text() || null,
country: $('td:nth-child(4)', this).text() || 'France',
uid: student.uid,
firstName: student.firstName,
lastName: student.lastName,
heticEmail: student.heticEmail,
email: student.email,
phone: student.phone,
cursus: student.cursus,
birthdate: student.birthdate,
picture: student.picture,
promotion: student.promotion,
enterprise: student.enterprise,
country: student.country,
};
students.push(studentObj);
// logger.debug(studentObj);
});
});
};
@ -94,5 +128,4 @@ export const action = async ({ req, logger }) => {
global.data = {};
global.data.students = students;
logger.info(`Harvested ${students.length} students`);
// logger.debug(global.data);
};