scores/odea
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Suppression script cron et amélioration enrichissement

Browse Source
This commit is contained in:
Michael RICOIS 2015-01-22 16:41:50 +00:00
parent 16a517780c
commit a1b9e2bb73
3 changed files with 150 additions and 205 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
scripts/build/configure.php
View File

@ -58,12 +58,7 @@ if(isset($opts->install))
mkdir(APPLICATION_PATH . '/../data');
mkdir(APPLICATION_PATH . '/../data/sessions');
//Initialisation du cache des valeurs des champs
//Modification des permissions
passthru('chown -R www-data: '.APPLICATION_PATH.'/../');
passthru('chmod +x '.APPLICATION_PATH.'/../scripts/cron.php');
passthru('chmod +x '.APPLICATION_PATH.'/../scripts/jobs/*.php');
}

71
scripts/cron.php
View File

@ -1,71 +0,0 @@
#!/usr/bin/php
<?php
// Define path to application directory
defined('APPLICATION_PATH')
|| define('APPLICATION_PATH', realpath(dirname(__FILE__) . '/../application'));
// Define application environment
define('APPLICATION_ENV', 'production');
// Ensure library/ is on include_path
set_include_path(implode(PATH_SEPARATOR, array(
realpath(APPLICATION_PATH . '/../library'),
get_include_path(),
)));
/** Zend_Application */
require_once 'Zend/Application.php';
// Create application, bootstrap, and run
$application = new Zend_Application(
APPLICATION_ENV,
APPLICATION_PATH . '/configs/application.ini'
);
try {
$opts = new Zend_Console_Getopt(
//Options
array(
'help|?' => "Aide.",
)
);
$opts->parse();
} catch (Zend_Console_Getopt_Exception $e) {
echo $e->getUsageMessage();
exit;
}
//Usage
if(isset($opts->help))
{
echo $opts->getUsageMessage();
exit;
}
$c = new Zend_Config($application->getOptions());
$db = Zend_Db::factory($c->profil->db->metier);
Zend_Db_Table::setDefaultAdapter($db);
$commandesM = new Application_Model_CiblageEnrichissementIdentifiants();
$sql = $commandesM->select()
->where('idProfil != ?', 0)
->where("dateStart != '0000-00-00 00:00:00'")
->where("dateStop = '0000-00-00 00:00:00'");
$result = $commandesM->fetchAll($sql);
if (count($result)>0){
exit;
}
//Si pas de traitement en cours alors on lance
$sql = $commandesM->select()
->where('idProfil != ?', 0)
->where("dateStart = '0000-00-00 00:00:00'")
->where("dateStop = '0000-00-00 00:00:00'")
->order('dateAdded ASC')->limit(1);
$result = $commandesM->fetchAll($sql);
if (count($result)>0) {
$info = $result->current();
echo "Lancement enrichissement $info->id\n";
exec('php '.realpath(dirname(__FILE__))."/jobs/enrichissement.php --id ".$info->id." &");
}

279
scripts/jobs/enrichissement.php
View File

@ -1,34 +1,37 @@
<?php
error_reporting(E_ALL ^ E_NOTICE ^ E_WARNING);
// Define path to application directory
// --- Define path to application directory
defined('APPLICATION_PATH')
|| define('APPLICATION_PATH', realpath(dirname(__FILE__) . '/../../application'));
// Define application environment
// --- Define application environment
define('APPLICATION_ENV', 'production');
// Ensure library/ is on include_path
// --- Ensure library/ is on include_path
set_include_path(implode(PATH_SEPARATOR, array(
realpath(APPLICATION_PATH . '/../library'),
get_include_path(),
)));
/** Zend_Application */
// --- Zend_Application
require_once 'Zend/Application.php';
// Create application, bootstrap, and run
// --- Create application, bootstrap, and run
$application = new Zend_Application(
APPLICATION_ENV,
APPLICATION_PATH . '/configs/application.ini'
);
// --- Console
try {
$opts = new Zend_Console_Getopt(
//Options
// --- Options
array(
'help|?' => "Aide.",
'id=s' => "Identifiant du traitement",
'file=s' => "Identifiant pour les traitements par fichier",
'cron' => "Lancement avec cron",
'verbose' => "Mode verbeux",
)
);
@ -38,15 +41,15 @@ try {
exit;
}
//Usage
if(isset($opts->help) || !isset($opts->id) && !isset($opts->file) )
// --- Usage
if( isset($opts->help) )
{
echo $opts->getUsageMessage(); exit;
}
$config = new Zend_Config($application->getOptions());
//Définition bdd
// --- Définition bdd
try {
$db = Zend_Db::factory($config->profil->db->metier);
Zend_Db_Table::setDefaultAdapter($db);
@ -54,32 +57,60 @@ try {
exit ( $e->getMessage() );
}
$liste = array();
$id = null;
if ($opts->cron)
{
$commandesM = new Application_Model_CiblageEnrichissementIdentifiants();
$sql = $commandesM->select()
->where('idProfil != ?', 0)
->where("dateStart != '0000-00-00 00:00:00'")
->where("dateStop = '0000-00-00 00:00:00'");
$result = $commandesM->fetchAll($sql);
if (count($result)>0){
exit;
}
//Si pas de traitement en cours alors on lance
$sql = $commandesM->select()
->where('idProfil != ?', 0)
->where("dateStart = '0000-00-00 00:00:00'")
->where("dateStop = '0000-00-00 00:00:00'")
->order('dateAdded ASC')->limit(1);
$result = $commandesM->fetchAll($sql);
if (count($result)>0) {
$info = $result->current();
echo date('Y-m-d H:i:s') . " - Lancement enrichissement $info->id\n";
$id = $info->id;
}
}
if ($opts->id)
{
//Read SIRETs
$id = $opts->id;
}
if ($id !== null)
{
// --- Read SIRETs
$commandesM = new Application_Model_CiblageEnrichissementIdentifiants();
$commande = $commandesM->find(intval($opts->id))->current();
$commande = $commandesM->find(intval($id))->current();
$identifiants = json_decode($commande->identifiants, true);
//Read profil for data extract
// --- Read profil for data extract
$profilM = new Application_Model_CiblageEnrichissementProfils();
$profil = $profilM->find(intval($commande->idProfil))->current();
$dataProfil = json_decode($profil->criteres, true);
}
else if ($opts->file)
{
}
//Something is needed
// --- Something is needed
if ( count($identifiants)==0 || count($dataProfil)==0 ) {
echo "Identifiants:".count($identifiants).", profil:".$count($profil)."\n";
//echo "Identifiants:".count($identifiants).", profil:".$count($profil)."\n";
exit;
}
//Let's go
// --- Let's go
$mois = substr($commande->dateAdded,0,4).substr($commande->dateAdded,5,2);
$path = $config->profil->path->data.'/'.$mois;
@ -90,37 +121,35 @@ require_once 'Scores/Enrichissement.php';
$dico = new Enrichissement();
$fields = $dico->getFields();
//Entete, Valeur de remplacement et Requete SQL
// --- Entete, Valeur de remplacement et Requete SQL
$tabEntete = array('siren', 'nic');
$tabEnteteLabel = array('SIREN', 'NIC');
$columns = array('LPAD(siren, 9, 000000000) AS siren', 'LPAD(nic,5,00000) AS nic');
$joins = array();
foreach ( $dataProfil as $item )
{
//Get item
if ( array_key_exists($item, $fields) )
{
foreach ( $dataProfil as $item ) {
// --- Get item
if ( array_key_exists($item, $fields) ) {
$field = $fields[$item];
//Définition de l'entete
// --- Définition de l'entete
$tabEnteteLabel[] = $field['label'];
$tabEntete[] = $item;
//Construction de la requete SQL
// --- Construction de la requete SQL
if ( array_key_exists('sql', $field) ) {
$columns[] = $field['sql'];
} else {
$columns[] = $field['column'].' AS '.$item;
}
//Pour les champs de type "code", ajouter le libellé
if ( array_key_exists('join', $field) )
{
// --- Pour les champs de type "code", ajouter le libellé
if ( array_key_exists('join', $field) ) {
$tabEnteteLabel[] = $field['join']['label'];
$joinColumn = $item.'Lib';
$tabEntete[] = $joinColumn;
//Sql
// --- Sql
$tableAlias = $item.'L';
$join['name'] = $field['join']['table'].' AS '.$tableAlias;
$join['col'] = $field['join']['column'].' AS '.$joinColumn;
@ -131,48 +160,17 @@ foreach ( $dataProfil as $item )
}
}
//Ajouter le champ presentRcs
// --- Ajouter le champ presentRcs
$columns[] = 'presentRcs';
$tabEntete[] = 'presentRcs';
$tabEnteteLabel[] = 'RCS';
$tabEnteteLabel[] = "Présent au RNCS";
//Paramètres du fichier
/**
* Possible file format
* csv, xlsx, xls, ods
* Possible options (for csv)
* separator : , ;
* encoding : ISO-8859-15, UTF-8
*/
$fileFormat = 'csv';
$fileOptions = array(
'separator' => ',',
'encoding' => 'UTF-8',
);
if ($commande->fileFormat == 'csv') {
$fileFormat = 'csv';
$options = json_decode($commande->fileOptions);
if (!empty($options->separator)) {
$fileOptions['separator'] = $options->separator;
}
if (!empty($options->encoding)) {
$fileOptions['encoding'] = $options->encoding;
}
}
if ($commande->fileFormat == 'xlsx') {
$fileFormat = 'xlsx';
}
//Ouverture fichier
// --- Ouverture fichier
if ($fileFormat == 'csv')
{
$outFile = $profil->login.'_'.$opts->id.'_'.date('YmdHis').'.csv';
$outFile = $profil->login.'_'.$id.'_'.date('YmdHis').'.csv';
$fp = fopen($path.'/'.$outFile, 'w');
//Ecrire l'entete
// ---- Ecrire l'entete
if (count($tabEnteteLabel) > 0)
{
fputcsv($fp, $tabEnteteLabel, $fileOptions['separator'], '"');
@ -181,10 +179,10 @@ if ($fileFormat == 'csv')
if ($fileFormat == 'xlsx')
{
$outFile = $profil->login.'_'.$opts->id.'_'.date('YmdHis').'.xlsx';
$outFile = $profil->login.'_'.$id.'_'.date('YmdHis').'.xlsx';
$objPHPExcel = new PHPExcel();
$sheet = $objPHPExcel->setActiveSheetIndex(0);
//Ecrire l'entete
// --- Ecrire l'entete
if (count($tabEnteteLabel) > 0)
{
foreach($tabEnteteLabel as $col => $value)
@ -194,103 +192,126 @@ if ($fileFormat == 'xlsx')
}
}
//Mise à jour des éléments
if ($opts->id)
{
$commandesM->update(array(
'dateStart'=>date('Y-m-d H:i:s'),
'fichier' => basename($outFile)
), "id = ".$commande->id);
// --- Mise à jour des éléments
if ($id !== null) {
$commandesM->update(array(
'dateStart'=> date('Y-m-d H:i:s'),
'fichier' => basename($outFile)
), "id = ".$commande->id);
}
//Pour chaque identifiant traiter les données
// --- Pour chaque identifiant traiter les données
$row = 1;
//Date de debut de traitement.
// --- Date de debut de traitement.
$dateStart = date('YmdHis');
$nbLineTotal = count($identifiants);
if ($opts->verbose) echo "Nb Lines :".$nbLineTotal."\n";
foreach ($identifiants as $siret)
{
if ($opts->verbose) echo "Line $row.\n";
if ($opts->verbose) echo "Line $row / $nbLineTotal : ";
$sql = $db->select()->from('etablissements_act', $columns, 'jo');
$sql->where("siren='".substr($siret,0,9)."' AND nic='".substr($siret,9,5)."'");
if (count($joins))
{
foreach ( $joins as $join )
{
if (count($joins)) {
foreach ( $joins as $join ) {
$sql->joinLeft($join['name'], $join['cond'], $join['col']);
}
}
$tabData = null;
// --- Get data in database
try {
$result = $db->fetchRow($sql);
$tabData = $db->fetchRow($sql, null, Zend_Db::FETCH_ASSOC);
} catch(Exception $e) {
echo $sql."\n";
echo date('Y-m-d H:i:s') . ' - Ligne '. $row . ' : Erreur ' . $sql . " : " . $e->getMessage() ."\n";
}
$tabData = $result->toArray();
//Trier pour la sortie
$tabSortie = array();
foreach($tabEntete as $key)
{
//Add static values
if ( array_key_exists($key, $fields) ) {
if ( array_key_exists('values', $fields[$key]) ) {
$valuesRef = $fields[$key]['values'];
$value = $tabData[$key];
//Remplace value if exist
if ( array_key_exists($value, $valuesRef) ) {
$tabData[$key] = $valuesRef[$tabData[$key]];
}
}
}
//Order data for CSV file
$tabSortie[] = isset($tabData[$key]) ? $tabData[$key] : '';
if ($opts->verbose) {
echo "IN=".$siret." : OUT=".$tabData['siren'];
}
// --- Ligne vide
if ( $tabData === false ) {
if ($opts->verbose) {
echo " - Ligne vide";
} else {
echo date('Y-m-d H:i:s') . ' - Ligne '. $row ." - Siret $siret introuvable dans la table.\n";
}
}
// --- Export des données
else {
// --- Trier pour la sortie
$tabSortie = array();
foreach($tabEntete as $key) {
// --- Add static values
if ( array_key_exists($key, $fields) ) {
if ( array_key_exists('values', $fields[$key]) ) {
$valuesRef = $fields[$key]['values'];
$value = $tabData[$key];
// --- Remplace value if exist
if ( array_key_exists($value, $valuesRef) ) {
$tabData[$key] = $valuesRef[$tabData[$key]];
}
}
}
if ( $key == 'presentRcs' ) {
$valuesRef = array(
'0' => "Non",
'1' => "Oui",
);
$value = $tabData[$key];
// --- Remplace value if exist
if ( array_key_exists($value, $valuesRef) ) {
$tabData[$key] = $valuesRef[$tabData[$key]];
}
}
// --- Order data for CSV file
$tabSortie[] = isset($tabData[$key]) ? $tabData[$key] : '';
}
// --- Ecriture dans le fichier
if ($fileFormat == 'csv') {
fputcsv($fp, $tabSortie, ',', '"');
}
if ($fileFormat == 'xlsx') {
foreach($tabSortie as $col => $value) {
$sheet->setCellValueByColumnAndRow($col, $row+1, $value);
}
}
}
//Ecriture dans le fichier
if ($fileFormat == 'csv')
{
fputcsv($fp, $tabSortie, ',', '"');
}
if ($fileFormat == 'xlsx')
{
foreach($tabSortie as $col => $value)
{
$sheet->setCellValueByColumnAndRow($col, $row+1, $value);
}
}
//Mise à jour des lignes traitées dans la base
if ($opts->id) {
// --- Mise à jour des lignes traitées dans la base
if ( $id !== null ) {
$commandesM->update(array('nbLigneTraites'=>$row), "id = ".$commande->id);
}
if ( $opts->verbose ) {
echo "\n";
}
$row++;
}
//Fermeture du fichier
if ($fileFormat == 'csv')
{
if ($fileFormat == 'csv') {
fclose($fp);
}
if ($fileFormat == 'xlsx')
{
if ($fileFormat == 'xlsx') {
$objWriter = PHPExcel_IOFactory::createWriter($objPHPExcel, 'Excel2007');
$objWriter->save($path.'/'.$outFile);
}
//Fin de création du fichier
if ($opts->id)
{
// --- Fin de création du fichier
if ($id !== null) {
$commandesM->update( array('dateStop' => date('Y-m-d H:i:s')) , "id = ".$commande->id);
}
/* == FUNCTION == */
/**