337 lines
11 KiB
PHP
337 lines
11 KiB
PHP
<?php
|
|
/**
|
|
* Récupération des fichiers de la DILA pour mise a disposition et suivi
|
|
*
|
|
* Compresser
|
|
* tar -xvf archive.tar [fichier1... ]
|
|
* tar czvf archive.tar.gz [fichier1... ]
|
|
* tar cjvf archive.tar.bz2 [fichier1... ]
|
|
* tar cJvf archive.tar.xz [fichier1... ]
|
|
*
|
|
* Extraire
|
|
* tar xzvf archive.tar.gz -C directory
|
|
* tar xjvf archive.tar.bz2 -C directory
|
|
* tar xJvf archive.tar.xz -C directory
|
|
*
|
|
* Vérifier
|
|
* tar -tf archive.tar
|
|
*
|
|
* Nettoyage
|
|
*
|
|
*/
|
|
error_reporting(E_ALL & ~E_STRICT & ~E_NOTICE & ~E_WARNING & ~E_DEPRECATED);
|
|
|
|
// --- Define path to application directory
|
|
defined('APPLICATION_PATH')
|
|
|| define('APPLICATION_PATH', realpath(__DIR__ . '/../application'));
|
|
|
|
// --- Define application environment
|
|
defined('APPLICATION_ENV')
|
|
|| define('APPLICATION_ENV', (getenv('APPLICATION_ENV') ? getenv('APPLICATION_ENV') : 'production'));
|
|
|
|
// --- Composer autoload
|
|
require_once realpath(__DIR__ . '/../vendor/autoload.php');
|
|
|
|
// --- Create application, bootstrap, and run
|
|
$application = new Zend_Application(APPLICATION_ENV, APPLICATION_PATH . '/configs/application.ini');
|
|
|
|
// --- Options
|
|
$displayUsage = false;
|
|
try {
|
|
$opts = new Zend_Console_Getopt(array(
|
|
'help|?' => "Displays usage information.",
|
|
'file-w' => "Télécharger à nouveau le fichier en mode manuel pour écrasement de l'ancien",
|
|
'type=w' => "Only look for one type of file",
|
|
'cron' => "Mandatory option for launch the cli in cron",
|
|
'from=w' => "restricted not opendata",
|
|
'date=w' => "Specify date",
|
|
'verbose|v' => "verbose mode",
|
|
));
|
|
$opts->parse();
|
|
} catch (Zend_Console_Getopt_Exception $e) {
|
|
$displayUsage = true;
|
|
}
|
|
|
|
// --- Aide / Options
|
|
if (count($opts->getOptions())==0 || isset($opts->help)) {
|
|
$displayUsage = true;
|
|
}
|
|
|
|
// --- Usage
|
|
if ($displayUsage) {
|
|
echo "Recuperation des fichiers de la DILA.\n";
|
|
echo $opts->getUsageMessage();
|
|
exit;
|
|
}
|
|
|
|
echo date('Y-m-d H:i:s') . " - Debut execution\n";
|
|
|
|
$c = new Zend_Config($application->getOptions());
|
|
$db = Zend_Db::factory($c->profil->db->metier);
|
|
Zend_Db_Table_Abstract::setDefaultAdapter($db);
|
|
|
|
// Database
|
|
$config = new \Doctrine\DBAL\Configuration();
|
|
$connectionParams = array(
|
|
'dbname' => $c->profil->db->metier->params->dbname,
|
|
'user' => $c->profil->db->metier->params->username,
|
|
'password' => $c->profil->db->metier->params->password,
|
|
'host' => $c->profil->db->metier->params->host,
|
|
'charset' => 'utf8',
|
|
'driver' => 'pdo_mysql',
|
|
);
|
|
|
|
try {
|
|
$connDoctrine = \Doctrine\DBAL\DriverManager::getConnection($connectionParams, $config);
|
|
Zend_Registry::set('doctrine', $connDoctrine);
|
|
} catch (\Doctrine\DBAL\DBALException $e) {
|
|
echo "Connection Database impossible.\n";
|
|
exit;
|
|
}
|
|
|
|
$STORAGEBASE = "/home/scores/batch/shared/sources/dila";
|
|
|
|
if ($opts->from == 'restricted') {
|
|
$FTPSERVER = "ftp.journal-officiel.gouv.fr";
|
|
$FTPUSER = "SCORE";
|
|
$FTPPASS = "SD075";
|
|
} else {
|
|
$FTPSERVER = "echanges.dila.gouv.fr";
|
|
$FTPUSER = "anonymous";
|
|
$FTPPASS = "guest";
|
|
}
|
|
|
|
// --- Liste des Types
|
|
$types = array(
|
|
'ASSOCIATION' => array(
|
|
'FTPDIR' => "ASSOCIATIONS/ASS_{YEAR}",
|
|
'PATTERN' => array(
|
|
'ASS[0-9]{8}.taz',
|
|
),
|
|
'STORAGE' => "associations/{YEAR}",
|
|
),
|
|
'BALO' => array(
|
|
'FTPDIR' => "BALO/{YEAR}",
|
|
'PATTERN' => array(
|
|
'balo_[0-9]{11}.taz',
|
|
),
|
|
'STORAGE' => "balo/{YEAR}",
|
|
),
|
|
'BOAMP' => array(
|
|
'FTPDIR' => "BOAMP/{YEAR}/apres_2_mars_2015",
|
|
'PATTERN' => array(
|
|
'BOAMP-J-AO_[0-9]{4}_[0-9]{6}.taz',
|
|
'BOAMP-J-IC-AA_[0-9]{4}_[0-9]{6}.taz',
|
|
'BOAMP-N-AO_[0-9]{4}_[0-9]{6}.taz',
|
|
'BOAMP-N-IC-AA_[0-9]{4}_[0-9]{6}.taz',
|
|
'MAPA-AO_[0-9]{4}_[0-9]{6}.taz',
|
|
'MAPA-IC-AA_[0-9]{4}_[0-9]{6}.taz',
|
|
),
|
|
'STORAGE' => "boamp/{YEAR}",
|
|
),
|
|
'BODACC' => array(
|
|
'FTPDIR' => "BODACC/{YEAR}",
|
|
'PATTERN' => array(
|
|
'RCS-B_BXB[0-9]{8}.taz', //RCS-B_BXB20160001.taz
|
|
'RCS-A_BXA[0-9]{8}.taz', //RCS-A_BXA20160001.taz
|
|
'PCL_BXA[0-9]{8}.taz', //PCL_BXA20160001.taz
|
|
'BILAN_BXC[0-9]{8}.taz',
|
|
'DIVA[0-9]{12}.taz',
|
|
),
|
|
'STORAGE' => "bodacc/{YEAR}",
|
|
),
|
|
'ASSOCIATIONCPT' => array(
|
|
'FTPDIR' => "COMPTES_DES_ASSOCIATIONS/FLUX",
|
|
'PATTERN' => array(
|
|
'cptassoc_[0-9]{6}_[0-9]{2}h[0-9]{2}.tar.gz',
|
|
),
|
|
'STORAGE' => "associationcpt/{YEAR}",
|
|
),
|
|
);
|
|
|
|
$oneShot = false;
|
|
if ($opts->file) {
|
|
$oneShot = true;
|
|
}
|
|
|
|
if ($opts->type) {
|
|
if (array_key_exists(strtoupper($opts->type), $types)) {
|
|
$types = array( $opts->type => $types[strtoupper($opts->type)] );
|
|
$oneShot = true;
|
|
} else {
|
|
echo date('Y-m-d H:i:s') . " - Type inexistant\n";
|
|
exit;
|
|
}
|
|
}
|
|
|
|
if ($opts->cron || $oneShot) {
|
|
|
|
if ($opts->cron) {
|
|
$execRef = new Scores_Exec_Ref('DILA');
|
|
$execId = $execRef->start();
|
|
}
|
|
|
|
// --- Connexion au FTP
|
|
$conn = ftp_connect($FTPSERVER);
|
|
if ( false === $conn) {
|
|
echo date('Y-m-d H:i:s') . " - Impossible de se connecter à $FTPSERVER\n";
|
|
exit;
|
|
}
|
|
|
|
// --- Login
|
|
$login = ftp_login($conn, $FTPUSER, $FTPPASS);
|
|
if ( false === $login) {
|
|
echo date('Y-m-d H:i:s') . " - Erreur authentication à $FTPSERVER\n";
|
|
exit;
|
|
}
|
|
|
|
// --- Pour chaque type
|
|
foreach ($types as $type => $typeParams) {
|
|
// --- Spécification répertoires
|
|
$dateDir = date('Y');
|
|
if ($opts->date) {
|
|
$dateDir = $opts->date;
|
|
}
|
|
if ($opts->from == 'restricted') {
|
|
$ftpDir = '';
|
|
} else {
|
|
$ftpDir = $typeParams['FTPDIR'];
|
|
$ftpDir = str_replace('{YEAR}', $dateDir, $ftpDir);
|
|
}
|
|
$storageDir = $typeParams['STORAGE'];
|
|
$storageDir = str_replace('{YEAR}', $dateDir, $storageDir);
|
|
$storageDir = $STORAGEBASE . '/' . $storageDir;
|
|
if (!file_exists($storageDir)) {
|
|
mkdir($storageDir, 0777, true);
|
|
}
|
|
|
|
// --- Liste des fichiers dans le répertoire
|
|
$contents = ftp_nlist($conn, $ftpDir);
|
|
|
|
if ($contents === false) {
|
|
echo date('Y-m-d H:i:s') . " - Impossible de lister le répertoire $ftpDir\n";
|
|
continue;
|
|
}
|
|
|
|
$unitTotal = count($contents);
|
|
$unitInc = $unitExec = 0;
|
|
if ($unitTotal > 0) {
|
|
foreach ($contents as $file) {
|
|
if ($opts->verbose) echo date('Y-m-d H:i:s') . " - $file\n";
|
|
// --- File pattern
|
|
if ($opts->from == 'restricted') {
|
|
$inPattern = false;
|
|
foreach ($typeParams['PATTERN'] as $p) {
|
|
if (preg_match('/^'.$p.'$/', $file)) {
|
|
$inPattern = true; break;
|
|
}
|
|
}
|
|
if ($inPattern === false) {
|
|
continue;
|
|
}
|
|
}
|
|
|
|
$downloadFile = false;
|
|
|
|
echo date('Y-m-d H:i:s') . " - Lecture $file\n";
|
|
$fileSql = "SELECT * FROM sdv1.flux_dila WHERE file=:file ORDER BY dateInsert DESC LIMIT 0,1";
|
|
$fileStmt = $connDoctrine->prepare($fileSql);
|
|
$fileStmt->bindValue('file', $file);
|
|
$fileStmt->execute();
|
|
|
|
// --- Récupération des infos FTP
|
|
$ftpDate = 0;
|
|
$fileTime = ftp_mdtm($conn, $ftpDir.'/'.$file);
|
|
if ($fileTime != -1) {
|
|
date_default_timezone_set('UTC');
|
|
$ftpDate = date('YmdHis', $fileTime);
|
|
date_default_timezone_set('Europe/Paris');
|
|
}
|
|
$ftpSize = ftp_size($conn, $ftpDir.'/'.$file);
|
|
|
|
// -- Fichier non présent en base
|
|
if ($fileStmt->rowCount()) {
|
|
$unitInc++;
|
|
$downloadFile = true;
|
|
// --- Le fichier existe déjà sur le système de fichier
|
|
if (file_exists($storageDir.'/'.$file)) {
|
|
echo date('Y-m-d H:i:s') . " - Le fichier existe $file\n";
|
|
// --- Enregistrement dans la base de données
|
|
$data = array(
|
|
'type' => $type,
|
|
'file' => $file,
|
|
'directory' => $storageDir,
|
|
'ftpSize' => $ftpSize,
|
|
'ftpDate' => $ftpDate,
|
|
'dateInsert'=> date('YmdHis'),
|
|
);
|
|
try {
|
|
$connDoctrine->insert('sdv1.flux_dila', $data);
|
|
}
|
|
catch(\Doctrine\DBAL\DBALException $e) {
|
|
echo date('Y-m-d H:i:s') . " - Erreur insertion bdd $file = ".$e->getMessage()."\n";
|
|
}
|
|
$downloadFile = false;
|
|
}
|
|
}
|
|
// --- Fichier déjà présent en base
|
|
else {
|
|
$fileRow = $fileStmt->fetch(\PDO::FETCH_OBJ);
|
|
if ($ftpSize != filesize($storageDir.'/'.$file) && $ftpSize != $fileRow->ftpSize) {
|
|
rename($storageDir.'/'.$file, $storageDir.'/'.$file.'_'.date('YmdHis'));
|
|
$downloadFile = true;
|
|
}
|
|
/*if ($ftpDate != date('YmdHis', filectime($storageDir.'/'.$file))) {
|
|
rename($storageDir.'/'.$file, $storageDir.'/'.$file.'_'.date('YmdHis'));
|
|
$downloadFile = true;
|
|
}*/
|
|
}
|
|
|
|
// --- Téléchargement du fichier
|
|
if ($downloadFile === true) {
|
|
echo date('Y-m-d H:i:s') . " - Telechargement $file\n";
|
|
if (ftp_get($conn, $storageDir.'/'.$file, $ftpDir.'/'.$file, FTP_BINARY)) {
|
|
echo date('Y-m-d H:i:s') . " - Ecriture du fichier $storageDir/$file\n";
|
|
// --- Vérification cohérance
|
|
|
|
// --- Enregistrement dans la base de données
|
|
$data = array(
|
|
'type' => $type,
|
|
'file' => $file,
|
|
'directory' => $storageDir,
|
|
'ftpSize' => $ftpSize,
|
|
'ftpDate' => $ftpDate,
|
|
'dateInsert'=> date('YmdHis'),
|
|
);
|
|
try {
|
|
$connDoctrine->insert('sdv1.flux_dila', $data);
|
|
}
|
|
catch(\Doctrine\DBAL\DBALException $e) {
|
|
echo date('Y-m-d H:i:s') . " - Erreur insertion bdd $file = ".$e->getMessage()."\n";
|
|
}
|
|
|
|
$unitExec++;
|
|
if ($execId !== null) {
|
|
$execRef->increment($execId, $unitExec);
|
|
}
|
|
|
|
} else {
|
|
echo date('Y-m-d H:i:s') . " - Impossible de télécharger $file\n";
|
|
}
|
|
}
|
|
|
|
}
|
|
}
|
|
}
|
|
|
|
// --- Fermeture du FTP
|
|
ftp_close($conn);
|
|
|
|
if ($execId !== null) {
|
|
$execRef->total($execId, $unitInc);
|
|
$execRef->end($execId);
|
|
}
|
|
}
|
|
|
|
echo date('Y-m-d H:i:s') . " - Fin execution\n";
|