batch/bin/sourceInsee.php

<?php
/**
 * Récupération des fichiers INSEE pour mise a disposition et suivi
 *
 */
// --- Define path to application directory
defined('APPLICATION_PATH')
    || define('APPLICATION_PATH', realpath(__DIR__ . '/../application'));

// --- Define application environment
defined('APPLICATION_ENV')
    || define('APPLICATION_ENV', (getenv('APPLICATION_ENV') ? getenv('APPLICATION_ENV') : 'production'));

// --- Composer autoload
require_once realpath(__DIR__ . '/../vendor/autoload.php');

// --- Create application, bootstrap, and run
$application = new Zend_Application(APPLICATION_ENV, APPLICATION_PATH . '/configs/application.ini');

// --- Options
$displayUsage = false;
try {
    $opts = new Zend_Console_Getopt(array(
        'help|?'    => "Displays usage information.",
        'file-w'    => "Télécharger à nouveau le fichier en mode manuel pour écrasement de l'ancien",
        'type=w'    => "Only look for one type of file",
        'cron'      => "Mandatory option for launch the cli in cron",
        'from=w'    => "restricted not opendata"
    ));
    $opts->parse();
} catch (Zend_Console_Getopt_Exception $e) {
    $displayUsage = true;
}

// --- Aide / Options
if (count($opts->getOptions())==0 || isset($opts->help)) {
    $displayUsage = true;
}

// --- Usage
if ($displayUsage) {
    echo "Recuperation des fichiers INSEE.\n";
    echo $opts->getUsageMessage();
    exit;
}

$BASEPATH = realpath(APPLICATION_PATH . '/../');
$STORAGEBASE = "/home/scores/batch/shared/sources/insee";
$FTP_URL = "ftp://81.255.68.110:21/download/";
$FTP_OPTION = "-k --ssl --disable-epsv --user client_syracuse23:cLiEnT_SYRAcuse23*37654 --cacert $BASEPATH/certs/ca.pem --cert $BASEPATH/certs/client.pem --key $BASEPATH/certs/key.pem";

echo date('Y-m-d H:i:s') . " - Debut execution\n";

$c = new Zend_Config($application->getOptions());
$db = Zend_Db::factory($c->profil->db->metier);
Zend_Db_Table_Abstract::setDefaultAdapter($db);

// Database
$config = new \Doctrine\DBAL\Configuration();
$connectionParams = array(
    'dbname'   => $c->profil->db->metier->params->dbname,
    'user'     => $c->profil->db->metier->params->username,
    'password' => $c->profil->db->metier->params->password,
    'host'     => $c->profil->db->metier->params->host,
    'charset'  => 'utf8',
    'driver'   => 'pdo_mysql',
);

try {
    $conn = \Doctrine\DBAL\DriverManager::getConnection($connectionParams, $config);
    Zend_Registry::set('doctrine', $conn);
} catch (\Doctrine\DBAL\DBALException $e) {
    echo "Connection Database impossible.\n";
    exit;
}

$storageDir = $STORAGEBASE . '/' . date('Y');
if (!file_exists($storageDir)) {
    mkdir($storageDir, 0777, true);
}

$oneShot = false;
if ($opts->file) {
    $oneShot = true;
}

if (file_exists('insee.lock')) {
    echo date('Y-m-d H:i:s') . "Fichier lock.\n";
    exit;
}

if ($opts->cron || $oneShot) {

    $execRef = new Scores_Exec_Ref('INSEE');
    $execId = $execRef->start();

    file_put_contents('insee.lock', "");

    /*
    $command = "curl $FTP_OPTION $FTP_URL";
    exec($command, $contents);
    print_r($contents); exit;
    */

    // --- Liste des fichiers
    $command = "curl -s $FTP_OPTION $FTP_URL | grep -e '^-' | awk '{ print $9 }'";
    exec($command, $contents);
    if (count($contents) == 0) {
        echo date('Y-m-d H:i:s') . " - Impossible de lister le répertoire.\n";
    }

    // --- Traitement pour chaque fichier
    $unitTotal = count($contents);
    $unitInc = $unitExec = 0;
    if ($unitTotal > 0) {
        foreach ($contents as $file) {
            echo date('Y-m-d H:i:s') . " - Lecture $file\n";
            $fileSql = "SELECT * FROM sdv1.flux_insee WHERE file=:file ORDER BY dateInsert DESC LIMIT 0,1";
            $fileStmt = $conn->prepare($fileSql);
            $fileStmt->bindValue('file', $file);
            $fileStmt->execute();
            $fileToDownload = false;
            // -- Fichier non présent en base
            if ($fileStmt->rowCount() == 0) {
                $unitInc++;
                $fileToDownload = true;
            }
            // --- Fichier déjà présent en base
            else {
                $fileRow = $fileStmt->fetch(\PDO::FETCH_OBJ);
                echo date('Y-m-d H:i:s') . " - Fichier $file en base";
                if (file_exists($fileRow->directory.'/'.$fileRow->file)) {
                    echo " exist\n";
                } else {
                    $unitInc++;
                    $fileToDownload = true;
                    echo " non exist\n";
                }
            }

            if ($fileToDownload) {
                // --- Téléchargement du fichier
                echo date('Y-m-d H:i:s') . " - Telechargement $file\n";
                $command = "cd $storageDir; curl -s -S $FTP_OPTION -O " . $FTP_URL . $file;
                exec($command);
                // --- Test integrity
                exec('unzip -t '.$storageDir.'/'.$file, $output, $integrity);
                if ($integrity == 0 && file_exists($storageDir.'/'.$file)) {
                    $type = 'insee';
                    $ftpSize = filesize($storageDir.'/'.$file);
                    $ftpDate = date ("YmdHis", filemtime($storageDir.'/'.$file));

                    echo date('Y-m-d H:i:s') . " - Ecriture du fichier $storageDir/$file\n";
                    // --- Enregistrement dans la base de données
                    $data = array(
                        'type'      => $type,
                        'file'      => $file,
                        'directory' => $storageDir,
                        'ftpSize'   => $ftpSize,
                        'ftpDate'   => $ftpDate,
                        'dateInsert'=> date('YmdHis'),
                    );
                    if ($fileStmt->rowCount() == 0) {
                        try {
                            $conn->insert('sdv1.flux_insee', $data);
                        }
                        catch(\Doctrine\DBAL\DBALException $e) {
                            echo date('Y-m-d H:i:s') . " - Erreur insertion $file = ".$e->getMessage()."\n";
                        }
                    }
                    else {
                        try {
                            $conn->update('sdv1.flux_insee', $data, array('id' => $fileRow->id));
                        }
                        catch(\Doctrine\DBAL\DBALException $e) {
                            echo date('Y-m-d H:i:s') . " - Erreur update $file = ".$e->getMessage()."\n";
                        }
                    }
                    $unitExec++;
                    if ($execId !== null) {
                        $execRef->increment($execId, $unitExec);
                    }
                }
                else {
                    echo date('Y-m-d H:i:s') . " - Impossible de télécharger $file\n";
                }
            }
        }
    }
    unlink('insee.lock');

    if ($execId !== null) {
        $execRef->total($execId, $unitInc);
        $execRef->end($execId);
    }
}

echo date('Y-m-d H:i:s') . " - Fin execution\n";