278 lines
9.6 KiB
PHP
278 lines
9.6 KiB
PHP
<?php
|
|
/**
|
|
* Récupération des fichiers de la DILA pour mise a disposition et suivi
|
|
*
|
|
*/
|
|
error_reporting(E_ALL & ~E_STRICT & ~E_NOTICE & ~E_WARNING & ~E_DEPRECATED);
|
|
|
|
// --- Define path to application directory
|
|
defined('APPLICATION_PATH')
|
|
|| define('APPLICATION_PATH', realpath(__DIR__ . '/../application'));
|
|
|
|
// --- Define application environment
|
|
defined('APPLICATION_ENV')
|
|
|| define('APPLICATION_ENV', (getenv('APPLICATION_ENV') ? getenv('APPLICATION_ENV') : 'production'));
|
|
|
|
// --- Composer autoload
|
|
require_once realpath(__DIR__ . '/../vendor/autoload.php');
|
|
|
|
// --- Create application, bootstrap, and run
|
|
$application = new Zend_Application(APPLICATION_ENV, APPLICATION_PATH . '/configs/application.ini');
|
|
|
|
// --- Options
|
|
$displayUsage = false;
|
|
try {
|
|
$opts = new Zend_Console_Getopt(array(
|
|
'help|?' => "Displays usage information.",
|
|
'file-w' => "Télécharger à nouveau le fichier en mode manuel pour écrasement de l'ancien",
|
|
'type=w' => "Only look for one type of file",
|
|
'cron' => "Mandatory option for launch the cli in cron",
|
|
'from=w' => "restricted not opendata",
|
|
'date=w' => "Specify date",
|
|
'verbose|v' => "verbose mode",
|
|
));
|
|
$opts->parse();
|
|
} catch (Zend_Console_Getopt_Exception $e) {
|
|
$displayUsage = true;
|
|
}
|
|
|
|
// --- Aide / Options
|
|
if (count($opts->getOptions())==0 || isset($opts->help)) {
|
|
$displayUsage = true;
|
|
}
|
|
|
|
// --- Usage
|
|
if ($displayUsage) {
|
|
echo "Recuperation des fichiers de la DILA.\n";
|
|
echo $opts->getUsageMessage();
|
|
exit;
|
|
}
|
|
|
|
echo date('Y-m-d H:i:s') . " - Debut execution\n";
|
|
|
|
$c = new Zend_Config($application->getOptions());
|
|
$db = Zend_Db::factory($c->profil->db->metier);
|
|
Zend_Db_Table_Abstract::setDefaultAdapter($db);
|
|
|
|
$STORAGEBASE = "/home/scores/batch/shared/sources/dila";
|
|
|
|
if ($opts->from == 'restricted') {
|
|
$FTPSERVER = "ftp.journal-officiel.gouv.fr";
|
|
$FTPUSER = "SCORE";
|
|
$FTPPASS = "SD075";
|
|
} else {
|
|
$FTPSERVER = "echanges.dila.gouv.fr";
|
|
$FTPUSER = "anonymous";
|
|
$FTPPASS = "guest";
|
|
}
|
|
|
|
// --- Liste des Types
|
|
$types = array(
|
|
'ASSOCIATION' => array(
|
|
'FTPDIR' => "ASSOCIATIONS/ASS_{YEAR}",
|
|
'PATTERN' => array(
|
|
'ASS[0-9]{8}.taz',
|
|
),
|
|
'STORAGE' => "associations/{YEAR}",
|
|
),
|
|
'BALO' => array(
|
|
'FTPDIR' => "BALO/{YEAR}",
|
|
'PATTERN' => array(
|
|
'balo_[0-9]{11}.taz',
|
|
),
|
|
'STORAGE' => "balo/{YEAR}",
|
|
),
|
|
'BOAMP' => array(
|
|
'FTPDIR' => "BOAMP/{YEAR}/apres_2_mars_2015",
|
|
'PATTERN' => array(
|
|
'BOAMP-J-AO_[0-9]{4}_[0-9]{6}.taz',
|
|
'BOAMP-J-IC-AA_[0-9]{4}_[0-9]{6}.taz',
|
|
'BOAMP-N-AO_[0-9]{4}_[0-9]{6}.taz',
|
|
'BOAMP-N-IC-AA_[0-9]{4}_[0-9]{6}.taz',
|
|
'MAPA-AO_[0-9]{4}_[0-9]{6}.taz',
|
|
'MAPA-IC-AA_[0-9]{4}_[0-9]{6}.taz',
|
|
),
|
|
'STORAGE' => "boamp/{YEAR}",
|
|
),
|
|
'BODACC' => array(
|
|
'FTPDIR' => "BODACC/{YEAR}",
|
|
'PATTERN' => array(
|
|
'RCS-B_BXB[0-9]{8}.taz', //RCS-B_BXB20160001.taz
|
|
'RCS-A_BXA[0-9]{8}.taz', //RCS-A_BXA20160001.taz
|
|
'PCL_BXA[0-9]{8}.taz', //PCL_BXA20160001.taz
|
|
'BILAN_BXC[0-9]{8}.taz',
|
|
'DIVA[0-9]{12}.taz',
|
|
),
|
|
'STORAGE' => "bodacc/{YEAR}",
|
|
),
|
|
'ASSOCIATIONCPT' => array(
|
|
'FTPDIR' => "COMPTES_DES_ASSOCIATIONS/FLUX",
|
|
'PATTERN' => array(
|
|
'cptassoc_[0-9]{6}_[0-9]{2}h[0-9]{2}.tar.gz',
|
|
),
|
|
'STORAGE' => "associationcpt/{YEAR}",
|
|
),
|
|
);
|
|
|
|
$fileM = new Application_Model_Sdv1FluxDila();
|
|
|
|
$oneShot = false;
|
|
if ($opts->file) {
|
|
$oneShot = true;
|
|
}
|
|
|
|
if ($opts->type) {
|
|
if (array_key_exists(strtoupper($opts->type), $types)) {
|
|
$types = array( $opts->type => $types[strtoupper($opts->type)] );
|
|
$oneShot = true;
|
|
} else {
|
|
echo date('Y-m-d H:i:s') . " - Type inexistant\n";
|
|
exit;
|
|
}
|
|
}
|
|
|
|
if ($opts->cron || $oneShot) {
|
|
|
|
// --- Connexion au FTP
|
|
$conn = ftp_connect($FTPSERVER);
|
|
if ( false === $conn) {
|
|
echo date('Y-m-d H:i:s') . " - Impossible de se connecter à $FTPSERVER\n";
|
|
exit;
|
|
}
|
|
|
|
// --- Login
|
|
$login = ftp_login($conn, $FTPUSER, $FTPPASS);
|
|
if ( false === $login) {
|
|
echo date('Y-m-d H:i:s') . " - Erreur authentication à $FTPSERVER\n";
|
|
exit;
|
|
}
|
|
|
|
// --- Pour chaque type
|
|
foreach ($types as $type => $typeParams) {
|
|
// --- Spécification répertoires
|
|
$dateDir = date('Y');
|
|
if ($opts->date) {
|
|
$dateDir = $opts->date;
|
|
}
|
|
if ($opts->from == 'restricted') {
|
|
$ftpDir = '';
|
|
} else {
|
|
$ftpDir = $typeParams['FTPDIR'];
|
|
$ftpDir = str_replace('{YEAR}', $dateDir, $ftpDir);
|
|
}
|
|
$storageDir = $typeParams['STORAGE'];
|
|
$storageDir = str_replace('{YEAR}', $dateDir, $storageDir);
|
|
$storageDir = $STORAGEBASE . '/' . $storageDir;
|
|
if (!file_exists($storageDir)) {
|
|
mkdir($storageDir, 0777, true);
|
|
}
|
|
|
|
// --- Liste des fichiers dans le répertoire
|
|
$contents = ftp_nlist($conn, $ftpDir);
|
|
|
|
if ($contents === false) {
|
|
echo date('Y-m-d H:i:s') . " - Impossible de lister le répertoire $ftpDir\n";
|
|
continue;
|
|
}
|
|
|
|
if (count($contents) > 0) {
|
|
foreach ($contents as $file) {
|
|
if ($opts->verbose) echo date('Y-m-d H:i:s') . " - $file\n";
|
|
// --- File pattern
|
|
if ($opts->from == 'restricted') {
|
|
$inPattern = false;
|
|
foreach ($typeParams['PATTERN'] as $p) {
|
|
if (preg_match('/^'.$p.'$/', $file)) {
|
|
$inPattern = true; break;
|
|
}
|
|
}
|
|
if ($inPattern === false) {
|
|
continue;
|
|
}
|
|
}
|
|
|
|
$downloadFile = false;
|
|
|
|
echo date('Y-m-d H:i:s') . " - Lecture $file\n";
|
|
$fileSql = $fileM->select()->where('file=?', $file)->order('dateInsert DESC')->limit(1);
|
|
$fileRow = $fileM->fetchRow($fileSql);
|
|
|
|
// --- Récupération des infos FTP
|
|
$ftpDate = 0;
|
|
$fileTime = ftp_mdtm($conn, $ftpDir.'/'.$file);
|
|
if ($fileTime != -1) {
|
|
date_default_timezone_set('UTC');
|
|
$ftpDate = date('YmdHis', $fileTime);
|
|
date_default_timezone_set('Europe/Paris');
|
|
}
|
|
$ftpSize = ftp_size($conn, $ftpDir.'/'.$file);
|
|
|
|
// -- Fichier non présent en base
|
|
if ($fileRow === null) {
|
|
$downloadFile = true;
|
|
// --- Le fichier existe déjà sur le système de fichier
|
|
if (file_exists($storageDir.'/'.$file)) {
|
|
echo date('Y-m-d H:i:s') . " - Le fichier existe $file\n";
|
|
// --- Enregistrement dans la base de données
|
|
$data = array(
|
|
'type' => $type,
|
|
'file' => $file,
|
|
'directory' => $storageDir,
|
|
'ftpSize' => $ftpSize,
|
|
'ftpDate' => $ftpDate,
|
|
'dateInsert'=> date('YmdHis'),
|
|
);
|
|
try {
|
|
$fileM->insert($data);
|
|
} catch(Zend_Db_Exception $e) {
|
|
echo date('Y-m-d H:i:s') . " - Erreur insertion bdd $file = ".$e->getMessage()."\n";
|
|
}
|
|
$downloadFile = false;
|
|
}
|
|
}
|
|
// --- Fichier déjà présent en base
|
|
else {
|
|
if ($ftpSize != filesize($storageDir.'/'.$file) && $ftpSize != $fileRow->ftpSize) {
|
|
rename($storageDir.'/'.$file, $storageDir.'/'.$file.'_'.date('YmdHis'));
|
|
$downloadFile = true;
|
|
}
|
|
/*if ($ftpDate != date('YmdHis', filectime($storageDir.'/'.$file))) {
|
|
move($storageDir.'/'.$file, $storageDir.'/'.$file.'_'.date('YmdHis'));
|
|
$downloadFile = true;
|
|
}*/
|
|
}
|
|
|
|
// --- Téléchargement du fichier
|
|
if ( $downloadFile === true) {
|
|
echo date('Y-m-d H:i:s') . " - Telechargement $file\n";
|
|
if (ftp_get($conn, $storageDir.'/'.$file, $ftpDir.'/'.$file, FTP_BINARY)) {
|
|
echo date('Y-m-d H:i:s') . " - Ecriture du fichier $storageDir/$file\n";
|
|
// --- Enregistrement dans la base de données
|
|
$data = array(
|
|
'type' => $type,
|
|
'file' => $file,
|
|
'directory' => $storageDir,
|
|
'ftpSize' => $ftpSize,
|
|
'ftpDate' => $ftpDate,
|
|
'dateInsert'=> date('YmdHis'),
|
|
);
|
|
try {
|
|
$fileM->insert($data);
|
|
} catch(Zend_Db_Exception $e) {
|
|
echo date('Y-m-d H:i:s') . " - Erreur insertion bdd $file = ".$e->getMessage()."\n";
|
|
}
|
|
} else {
|
|
echo date('Y-m-d H:i:s') . " - Impossible de télécharger $file\n";
|
|
}
|
|
}
|
|
|
|
}
|
|
}
|
|
}
|
|
|
|
// --- Fermeture du FTP
|
|
ftp_close($conn);
|
|
}
|
|
|
|
echo date('Y-m-d H:i:s') . " - Fin execution\n";
|