From 98eb29e75342c793c99b3576edbd3b5d1797c71d Mon Sep 17 00:00:00 2001 From: sundowndev Date: Mon, 18 Feb 2019 23:20:03 +0100 Subject: [PATCH] Pull command --- commands/pull/index.js | 83 +++++++++++++++++++++++++++++------------- 1 file changed, 58 insertions(+), 25 deletions(-) diff --git a/commands/pull/index.js b/commands/pull/index.js index d3975af..c5ec450 100644 --- a/commands/pull/index.js +++ b/commands/pull/index.js @@ -4,7 +4,7 @@ export const description = 'Pull data from outils.hetic.net'; const rp = require("request-promise"); const cheerio = require('cheerio'); -export const action = async ({ req, logger }) => { +export const action = async ({ logger }) => { const options = { followAllRedirects: true, method: 'GET', @@ -12,7 +12,7 @@ export const action = async ({ req, logger }) => { headers: { 'cache-control': 'no-cache', 'User-Agent': 'User-Agent: Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:65.0) Gecko/20100101 Firefox/65.0', - 'Cookie': `PHPSESSID=${global.user.phpsessid};`, + 'Cookie': `${global.user.phpsessid};`, }, transform: function (body) { return cheerio.load(body); @@ -23,7 +23,6 @@ export const action = async ({ req, logger }) => { await rp(options) .then(function ($) { - // logger.log(options.headers); logger.info('Successfully fetched outils.hetic.net'); $('select#select_list_promo_students option').each(function (i, elem) { @@ -43,7 +42,7 @@ export const action = async ({ req, logger }) => { headers: { 'cache-control': 'no-cache', 'User-Agent': 'User-Agent: Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:65.0) Gecko/20100101 Firefox/65.0', - 'Cookie': `PHPSESSID=${global.user.phpsessid};`, + 'Cookie': `${global.user.phpsessid};`, }, transform: function (body) { return cheerio.load(body); @@ -52,41 +51,76 @@ export const action = async ({ req, logger }) => { await rp(options) .then(async ($) => { - // logger.debug($('body')); logger.info(`Fetching students from ${promotion} promotion...`); - await $('table.liste_insertions.liste_insertions_stage tbody tr').each(function (i, student) { + $('table.liste_insertions.liste_insertions_stage tbody tr').each(async function () { $('td a', this).each(function (index, col) { if ($(col).text().slice(-1) === ' ') $(col).text($(col).text().slice(0, -1)); else if ($(col).text() === ' ') $(col).text(''); }); - let uid = $('td:nth-child(1) a', this).attr('href').replace('/ogi/profil/', ''); - let lastName = $('td:nth-child(1)', this).text(); - let firstName = $('td:nth-child(2)', this).text(); - let heticEmail = `${firstName.toLowerCase()}.${lastName.toLowerCase()}@hetic.net` + const student = {}; + student.uid = $('td:nth-child(1) a', this).attr('href').replace('/ogi/profil/', ''); + student.lastName = $('td:nth-child(1)', this).text(); + student.firstName = $('td:nth-child(2)', this).text(); + student.heticEmail = `${student.firstName.toLowerCase()}.${student.lastName.toLowerCase()}@hetic.net` .normalize('NFD') .replace(/[\u0300-\u036f]/g, "") .replace(new RegExp(' ', 'g'), ''); + student.email = null; + student.birthdate = null + student.phone = null; + student.cursus = null; + student.picture = `https://outils.hetic.net/external/picture/${student.uid}`; + student.enterprise = $('td:nth-child(3)', this).text() || null; + student.country = $('td:nth-child(4)', this).text() || 'France'; + student.promotion = promotion; + + await rp({ + followAllRedirects: true, + method: 'GET', + url: 'https://outils.hetic.net/ogi/profil/' + student.uid, + headers: { + 'cache-control': 'no-cache', + 'User-Agent': 'User-Agent: Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:65.0) Gecko/20100101 Firefox/65.0', + 'Cookie': `${global.user.phpsessid};`, + }, + transform: function (body) { + return cheerio.load(body); + }, + }) + .then(function ($) { + let secondLi = $('.left .item:nth-child(1) ul li:nth-child(2)').text(); + + student.email = $('.left .item:nth-child(1) ul li:nth-child(1)').text() || null; + student.cursus = $('.right.right_first .infos h3').text().replace(` P${student.promotion}`, '') || null; + + if (secondLi.match(/^(?:n\é\(e\)\ le\ )(?:.*)/)) { + secondLi = secondLi.replace('né(e) le ', '') || null; + secondLi = secondLi.split('/'); + + student.birthdate = new Date(`${secondLi[2]}-${secondLi[1]}-${secondLi[0]}`); + } else if (secondLi.match(/^[0-9]{8,}/)) { + student.phone = secondLi || null; + } + }) let studentObj = { - uid: uid, - firstName: firstName, - lastName: lastName, - heticEmail: heticEmail, - email: null, - phone: null, - cursus: null, - birthdate: null, - picture: `https://outils.hetic.net/external/picture/${uid}`, - promotion: promotion, - enterprise: $('td:nth-child(3)', this).text() || null, - country: $('td:nth-child(4)', this).text() || 'France', + uid: student.uid, + firstName: student.firstName, + lastName: student.lastName, + heticEmail: student.heticEmail, + email: student.email, + phone: student.phone, + cursus: student.cursus, + birthdate: student.birthdate, + picture: student.picture, + promotion: student.promotion, + enterprise: student.enterprise, + country: student.country, }; students.push(studentObj); - - // logger.debug(studentObj); }); }); }; @@ -94,5 +128,4 @@ export const action = async ({ req, logger }) => { global.data = {}; global.data.students = students; logger.info(`Harvested ${students.length} students`); - // logger.debug(global.data); }; \ No newline at end of file