Re-share project

This commit is contained in:
Michael RICOIS 2013-10-08 09:05:29 +00:00
parent 5b3f2444cb
commit e7acccd1fc
8 changed files with 353 additions and 118 deletions

156
config/Sphinx/DOC Normal file
View File

@ -0,0 +1,156 @@
La base de données doit communiqué en UTF-8
sql_query_pre = SET NAMES utf8
charset_type = utf-8
# 'utf-8' defaults for English and Russian
charset_table = 0..9, A..Z->a..z, _, a..z, \
U+410..U+42F->U+430..U+44F, U+430..U+44F
# For french
charset_table = 0..9, A..Z->a..z, a..z, \
U+00C0..U+00D6->U+00E0..U+00F6, U+00E0..U+00F6, \
U+00D8..U+00DE->U+00F8..U+00FE, U+00F8..U+00FE, \
U+0152->U+0153, U+0153 \
Morphology
==========
# builtin preprocessors are 'none', 'stem_en', 'stem_ru', 'stem_enru',
# 'soundex', and 'metaphone'; additional preprocessors available from
# libstemmer are 'libstemmer_XXX', where XXX is algorithm code
# (see libstemmer_c/libstemmer/modules.txt)
morphology = libstemmer_french
Taille minimum des mots ou l'on applique la morphology
min_stemming_len = 4
According to libstimmer.c/libstimmer/modules.txt, the french module can be refered to by either
french, fr, fre, fra... french UTF_8,ISO_8859_1 french,fr,fre,fra
Utilisation des stopwords
=========================
une, le, la, les, de, du, dans, l', d', @
Utilisation des wordforms
=========================
& > ET
Les chiffres
un > 1
deux > 2
trois > 3
quatre > 4
cinq > 5
six > 6
sept > 7
huit > 8
neuf > 9
dix > 10
onze > 11
douze > 12
treize > 13
quatorze > 14
quinze > 15
seize > 16
vingt > 20
vingts > 20
trente > 30
quarante > 40
cinquante > 50
soixante > 60
quatrevingt > 80
cent > 100
cents > 100
mille > 1000
Les chiffres romain
Index DIR
=========
source dir
{
type = mysql
sql_host = 192.168.3.30
sql_user = sphinx
sql_pass = indexer
sql_db = jo
sql_query_pre =
sql_query = SELECT id, siren, actif, \
IF(civilite='M',1,IF(civilite='MME' OR civilite='MLLE',2,0)) AS genre, \
CONCAT_WS(' ',dirRS,nom,naissance_nom) AS nom, prenom, \
YEAR(naissance_date) AS naiss_annee, \
MONTH(naissance_date) AS naiss_mois, \
DAY(naissance_date) AS naiss_jour, \
naissance_lieu, adr_dep \
FROM rncs_dirigeants;
sql_attr_uint = naiss_annee
sql_attr_uint = naiss_mois
sql_attr_uint = naiss_jour
sql_attr_uint = siren
sql_attr_uint = actif
sql_attr_uint = genre
sql_attr_uint = adr_dep
}
index dir
{
source = dir
path = /dbs/sphinx/dir
docinfo = extern
morphology = none
charset_type = sbcs
charset_table = 0..9, A..Z->a..z, a..z, \
U+23, U+25, U+26, U+2B, U+3D, U+40, \
U+C0..U+DE->U+E0..U+FE, U+DF, U+E0..U+FF
}
Index ENT
=========
source ent
{
type = mysql
sql_host = 192.168.3.30
sql_user = sphinx
sql_pass = indexer
sql_db = jo
sql_query_pre =
sql_query = SELECT id, siren, nic, siege, raisonSociale, enseigne, sigle, identite_pre, \
CONCAT_WS(' ', REPLACE(raisonSociale,' ',''), REPLACE(enseigne,' ',''), \
REPLACE(sigle,' ',''),REPLACE(identite_pre,' ','')) AS nom, \
adr_dep, actif, adr_num, adr_typeVoie, CONCAT(adr_libVoie,' ',adr_comp) AS adresse, \
adr_cp, adr_ville AS ville, cj, ape_etab, (siren>200) AS sirenValide, tel, fax, rang \
FROM etablissements_tmp;
sql_attr_str2ordinal = adr_typeVoie
sql_attr_uint = siren
sql_attr_uint = nic
sql_attr_uint = siege
sql_attr_uint = adr_num
sql_attr_uint = adr_cp
sql_attr_uint = adr_dep
sql_attr_uint = actif
sql_attr_uint = cj
sql_attr_uint = sirenValide
sql_attr_uint = rang
}
index ent
{
source = source_ent
path = /dbs/sphinx/ent
docinfo = extern
morphology = libstemmer_french
wordforms = /usr/local/sphinx/etc/wordforms.txt
charset_type = sbcs
charset_table = 0..9, A..Z->a..z, a..z, \
U+23, U+25, U+26, U+2B, U+3D, U+40, \
U+C0..U+DE->U+E0..U+FE, U+DF, U+E0..U+FF
}

View File

@ -5,7 +5,7 @@ source act
sql_user = sphinx
sql_pass = indexer
sql_db = jo
sql_query_pre =
sql_query_pre = SET NAMES utf8
sql_query = \
SELECT id, Siren1, RaisonSociale, Siren2, actif, dateLien, IF(dateUpdate>dateInsert,dateUpdate,dateInsert) as dateMaj, \
source, Pmin*1 AS PminNum, MajMin, PpPm, IF(Pays='','FRA',Pays) AS Pays \
@ -25,8 +25,15 @@ index act
path = /dbs/sphinx/act
docinfo = extern
morphology = none
charset_type = sbcs
charset_type = utf-8
charset_table = 0..9, A..Z->a..z, a..z, \
U+23, U+25, U+26, U+2B, U+3D, U+40, \
U+C0..U+DE->U+E0..U+FE, U+DF, U+E0..U+FF
U+23, U+25, U+26, U+2B, U+3D, U+40, \
U+C0->a, U+C1->a, U+C2->a, U+C3->a, U+C4->a, U+C5->a, U+C6->a, U+C7->c, \
U+C8->e, U+C8->e, U+C9->e, U+CA->e, U+CB->e, U+CC->i, U+CD->i, U+CE->i, \
U+CF->i, U+D0->d, U+D1->n, U+D2->o, U+D3->o, U+D4->o, U+D5->o, U+D6->o, \
U+D8->o, U+D9->u, U+DA->u, U+DB->u, U+DC->u, U+DD->y, U+E0->a, U+E1->a, \
U+E2->a, U+E3->a, U+E4->a, U+E5->a, U+E7->c, U+E8->e, U+E9->e, U+EA->e, \
U+EB->e, U+EC->i, U+ED->i, U+EE->i, U+EF->i, U+F1->n, U+F2->o, U+F3->o, \
U+F4->o, U+F5->o, U+F6->o, U+F8->o, U+F9->u, U+FA->u,U+FB->u, U+FC->u, \
U+FD->y, U+FF->y, U+0152->U+0153, U+0153
}

View File

@ -5,7 +5,7 @@ source dir
sql_user = sphinx
sql_pass = indexer
sql_db = jo
sql_query_pre =
sql_query_pre = SET NAMES utf8
sql_query = \
SELECT id, siren, adr_dep, typeDir, dirSiren, civilite, CONCAT(nom,' ',naissance_nom, ' ', dirRS) AS nom, prenom, \
YEAR(naissance_date) AS naiss_annee, \
@ -30,8 +30,15 @@ index dir
path = /dbs/sphinx/dir
docinfo = extern
morphology = none
charset_type = sbcs
charset_type = utf8
charset_table = 0..9, A..Z->a..z, a..z, \
U+23, U+25, U+26, U+2B, U+3D, U+40, \
U+C0..U+DE->U+E0..U+FE, U+DF, U+E0..U+FF
U+23, U+25, U+26, U+2B, U+3D, U+40, \
U+C0->a, U+C1->a, U+C2->a, U+C3->a, U+C4->a, U+C5->a, U+C6->a, U+C7->c, \
U+C8->e, U+C8->e, U+C9->e, U+CA->e, U+CB->e, U+CC->i, U+CD->i, U+CE->i, \
U+CF->i, U+D0->d, U+D1->n, U+D2->o, U+D3->o, U+D4->o, U+D5->o, U+D6->o, \
U+D8->o, U+D9->u, U+DA->u, U+DB->u, U+DC->u, U+DD->y, U+E0->a, U+E1->a, \
U+E2->a, U+E3->a, U+E4->a, U+E5->a, U+E7->c, U+E8->e, U+E9->e, U+EA->e, \
U+EB->e, U+EC->i, U+ED->i, U+EE->i, U+EF->i, U+F1->n, U+F2->o, U+F3->o, \
U+F4->o, U+F5->o, U+F6->o, U+F8->o, U+F9->u, U+FA->u,U+FB->u, U+FC->u, \
U+FD->y, U+FF->y, U+0152->U+0153, U+0153
}

View File

@ -5,7 +5,7 @@ source ent
sql_user = sphinx
sql_pass = indexer
sql_db = jo
sql_query_pre =
sql_query_pre = SET NAMES utf8
sql_query = SELECT id, siren, nic, siege, CONCAT(raisonSociale,\
' "" ',enseigne,' "" ',sigle,' "" ',identite_pre,' "" ',\
REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(sigle,' ',''),\

View File

@ -7,7 +7,7 @@ indexer
{
# memory limit, in bytes, kiloytes (16384K) or megabytes (256M)
# optional, default is 32M, max is 2047M, recommended is 256M to 1024M
mem_limit = 2047M
mem_limit = 1024M
# maximum IO calls per second (for I/O throttling)
# optional, default is 0 (unlimited)

View File

@ -1,32 +1,42 @@
source source_act
source act
{
type = mysql
sql_host = 192.168.3.30
sql_user = sphinx
sql_pass = indexer
sql_db = jo
sql_query_pre =
sql_query = \
SELECT id, Siren1, RaisonSociale, Siren2, actif, dateLien, IF(dateUpdate>dateInsert,dateUpdate,dateInsert) as dateMaj, \
source, Pmin*1 AS PminNum, MajMin, PpPm, IF(Pays='','FRA',Pays) AS Pays \
FROM liens \
WHERE ActionPart=1 AND Siren1>1000 AND Siren1 IN (SELECT DISTINCT SIREN FROM etablissements_old);
sql_attr_uint = actif
sql_attr_uint = source
sql_attr_uint = PminNum
sql_attr_str2ordinal= MajMin
sql_attr_str2ordinal= PpPm
sql_attr_str2ordinal= Pays
type = mysql
sql_host = 192.168.3.30
sql_user = sphinx
sql_pass = indexer
sql_db = jo
sql_query_pre = SET NAMES utf8
sql_query = \
SELECT l.id, l.idPar, l.idAct, r.siren, r.actif, r.PpPm, r.RS, r.adresse_cp, r.adresse_ville, p.libPays AS pays, l.PDetention \
FROM liens2 l, liensRef r, tabPays p \
WHERE l.dateSuppr = '0000-00-00 00:00:00' \
AND r.id = l.idAct \
AND ( r.siren>1000 OR (r.siren=0 AND r.adresse_pays!='FRA') OR (r.siren=0 AND r.PpPm='PP') ) \
AND p.codPays3 = r.adresse_pays;
sql_attr_uint = idAct
sql_attr_uint = actif
sql_attr_float= PDetention
sql_attr_str2ordinal = pays
}
index act
{
source = source_act
source = act
path = /dbs/sphinx/act
docinfo = extern
morphology = none
charset_type = sbcs
charset_table = 0..9, A..Z->a..z, a..z, \
U+23, U+25, U+26, U+2B, U+3D, U+40, \
U+C0..U+DE->U+E0..U+FE, U+DF, U+E0..U+FF
}
U+23, U+25, U+26, U+2B, U+3D, U+40, \
U+C0->a, U+C1->a, U+C2->a, U+C3->a, U+C4->a, U+C5->a, U+C6->a, U+C7->c, \
U+C8->e, U+C8->e, U+C9->e, U+CA->e, U+CB->e, U+CC->i, U+CD->i, U+CE->i, \
U+CF->i, U+D0->d, U+D1->n, U+D2->o, U+D3->o, U+D4->o, U+D5->o, U+D6->o, \
U+D8->o, U+D9->u, U+DA->u, U+DB->u, U+DC->u, U+DD->y, U+E0->a, U+E1->a, \
U+E2->a, U+E3->a, U+E4->a, U+E5->a, U+E7->c, U+E8->e, U+E9->e, U+EA->e, \
U+EB->e, U+EC->i, U+ED->i, U+EE->i, U+EF->i, U+F1->n, U+F2->o, U+F3->o, \
U+F4->o, U+F5->o, U+F6->o, U+F8->o, U+F9->u, U+FA->u,U+FB->u, U+FC->u, \
U+FD->y, U+FF->y, U+0152->U+0153, U+0153
}

View File

@ -0,0 +1,65 @@
source dir
{
type = mysql
sql_host = 192.168.3.30
sql_user = sphinx
sql_pass = indexer
sql_db = jo
sql_query_pre = SET NAMES utf8
sql_query = SELECT id, siren, adr_dep, typeDir, dirSiren, civilite, CONCAT(nom,' ',naissance_nom, ' ', dirRS) AS nom, prenom, \
YEAR(naissance_date) AS naiss_annee, \
MONTH(naissance_date) AS naiss_mois, \
DAY(naissance_date) AS naiss_jour, \
naissance_lieu, fonction_code, actif \
FROM dirigeants_tmp;
sql_attr_str2ordinal= civilite
sql_attr_str2ordinal= typeDir
sql_attr_str2ordinal= fonction_code
sql_attr_uint = dirSiren
sql_attr_uint = naiss_annee
sql_attr_uint = naiss_mois
sql_attr_uint = naiss_jour
sql_attr_uint = actif
# sql_attr_uint = genre
sql_attr_uint = adr_dep
}
index dir
{
source = dir
path = /dbs/sphinx/dir
docinfo = extern
morphology = none
#ispell_aff = /usr/local/sphinx/etc/francais.aff
charset_type = utf-8
charset_table = 0..9, A..Z->a..z, a..z, \
U+23, U+25, U+26, U+2B, U+3D, U+40, \
U+C0->a, U+C1->a, U+C2->a, U+C3->a, U+C4->a, U+C5->a, U+C6->a, U+C7->c, \
U+C8->e, U+C8->e, U+C9->e, U+CA->e, U+CB->e, U+CC->i, U+CD->i, U+CE->i, \
U+CF->i, U+D0->d, U+D1->n, U+D2->o, U+D3->o, U+D4->o, U+D5->o, U+D6->o, \
U+D8->o, U+D9->u, U+DA->u, U+DB->u, U+DC->u, U+DD->y, U+E0->a, U+E1->a, \
U+E2->a, U+E3->a, U+E4->a, U+E5->a, U+E7->c, U+E8->e, U+E9->e, U+EA->e, \
U+EB->e, U+EC->i, U+ED->i, U+EE->i, U+EF->i, U+F1->n, U+F2->o, U+F3->o, \
U+F4->o, U+F5->o, U+F6->o, U+F8->o, U+F9->u, U+FA->u,U+FB->u, U+FC->u, \
U+FD->y, U+FF->y, U+0152->U+0153, U+0153
}
index dir_phx
{
source = dir
path = /dbs/sphinx/dir_phx
docinfo = extern
morphology = libstemmer_french
charset_type = utf-8
charset_table = 0..9, A..Z->a..z, a..z, \
U+23, U+25, U+26, U+2B, U+3D, U+40, \
U+C0->a, U+C1->a, U+C2->a, U+C3->a, U+C4->a, U+C5->a, U+C6->a, U+C7->c, \
U+C8->e, U+C8->e, U+C9->e, U+CA->e, U+CB->e, U+CC->i, U+CD->i, U+CE->i, \
U+CF->i, U+D0->d, U+D1->n, U+D2->o, U+D3->o, U+D4->o, U+D5->o, U+D6->o, \
U+D8->o, U+D9->u, U+DA->u, U+DB->u, U+DC->u, U+DD->y, U+E0->a, U+E1->a, \
U+E2->a, U+E3->a, U+E4->a, U+E5->a, U+E7->c, U+E8->e, U+E9->e, U+EA->e, \
U+EB->e, U+EC->i, U+ED->i, U+EE->i, U+EF->i, U+F1->n, U+F2->o, U+F3->o, \
U+F4->o, U+F5->o, U+F6->o, U+F8->o, U+F9->u, U+FA->u,U+FB->u, U+FC->u, \
U+FD->y, U+FF->y, U+0152->U+0153, U+0153
}

View File

@ -1,93 +1,83 @@
#!/usr/bin/php
<?php
Class configure
// Paramètres
if ( count($argc)<1 || in_array($argv[1], array('--help', '-help', '-h', '-?')) ) {
?>
Utilisation : <?php echo $argv[0]; ?> [actions]
Installation :
- install list
- install version
Configuration :
- config initscript
- config logrotate
- config list
- config server *.conf
<?php
exit;
}
$hostname = exec('echo $HOSTNAME');
$SPHINX_VERSION = array(
'0.9.9' => array('archive'=>'sphinx-0.9.9.tar.gz', 'dir'=>'sphinx-0.9.9'),
'1.10' => array('archive'=>'sphinx-1.10-beta.tar.gz', 'dir'=>'sphinx-1.10-beta'),
'2.0.2' => array('archive'=>'sphinx-2.0.2-beta.tar.gz', 'dir'=>'sphinx-2.0.2-beta'),
'2.0.3' => array('archive'=>'sphinx-2.0.3-release.tar.gz', 'dir'=>'sphinx-2.0.3-release'),
'2.0.4' => array('archive'=>'sphinx-2.0.4-release.tar.gz', 'dir'=>'sphinx-2.0.4-release'),
);
if ($argv[1]=='install')
{
protected $dirname;
protected $sphinx;
protected $hostname;
protected $sql;
protected $indexes;
public function __construct()
{
$this->hostname = exec('echo $(hostname)');
$this->sphinx = 'sphinx.conf';
$this->dirname = 'config/'.$this->hostname.'/';
(file_exists($this->sphinx))?unlink($this->sphinx):'';
//Version
if (!array_key_exists($argv[2], $SPHINX_VERSION)){
exit;
}
public function printIndexes($nbServer)
{
$dir = opendir($this->dirname.$this->sql[$nbServer].'/');
$i = 1;
while($file = readdir($dir)) {
$extension = explode('.', $file);
if($file != '.' and $file != '..' and $extension[1] == 'conf') {
echo "\t\t".'['.$i.'] '. $file."\n";
$this->indexes[$i] = $file;$i++;
}
}
if(count($this->indexes) == 0) {
echo "\tAucun indexes dans le repertoire : '".$this->dirname.$this->sql[$nbServer]."' \n";
exit;
}
}
public function printMysqlServer()
{
$dir = opendir($this->dirname);
$i = 1;
while($rep = readdir($dir)) {
if($rep != '.' and $rep != '..' and is_dir($this->dirname.$rep)) {
echo "\t".'['.$i.'] '.$rep."\n";
$this->sql[$i] = $rep;
$this->printIndexes($i);
$i++;
}
}
if(count($this->sql) == 0) {
echo "\tAucun serveurs dans le repertoire : '".$this->dirname."' \n";
exit;
}
}
public function createConfig($mySqlServer, $index)
{
$dirname = $this->dirname.$mySqlServer.'/';
$dir = opendir($dirname);
$version = $argv[2];
if(file_exists($dirname.$index)) {
file_put_contents($this->sphinx, file_get_contents($dirname.$index), FILE_APPEND);
return (true);
}
return (false);
//Compilation
$cmd = array();
$cmd[] = "tar xzvf ".$SPHINX_VERSION[$version]['archive'];
$cmd[] = "cp libstemmer_c.tgz ".$SPHINX_VERSION[$version]['dir']."/";
$cmd[] = "cd ".$SPHINX_VERSION[$version]['dir']."/";
$cmd[] = "tar xzvf libstemmer_c.tgz";
$cmd[] = "./configure --with-libstemmer --prefix=/usr/local/sphinx";
$cmd[] = "make";
$cmd[] = "make install";
foreach($cmd as $c){
passthru($c);
}
public function inviteDeCommande()
{
$handle = fopen('.', 'r');
$server = '';
fwrite(STDOUT, "\n - Serveurs SQL disponnible :\n");
$this->printMysqlServer();
while($server != "exit\n") {
echo "Selectionnez Server et Indexe : ";
$server = fgets(STDIN);
$indexes = explode(' ', $server);
if(isset($this->sql[intval($indexes[0])]) and isset($this->indexes[intval($indexes[1])])) {
$this->createConfig($this->sql[intval($indexes[0])], $this->indexes[intval($indexes[1])]);
echo 'Indexe ajouté !'."\n";
} else
echo 'Cet indexe n\'existe pas !'."\n";
}
file_put_contents($this->sphinx, file_get_contents($this->dirname.$this->sphinx), FILE_APPEND);
exec('sudo mv '.$this->sphinx.' /usr/local/sphinx/etc/'.$this->sphinx);
}
}
$a = new configure();
$a->inviteDeCommande();
if ($argv[1]=='config')
{
//Initscript
if ($argv[2]=='initscript') {
exec("cp -v initscript/sphinxsearch /etc/init.d/");
exit;
}
//Logrotate
if ($argv[2]=='logrotate') {
echo exec("which logrotate");
exec("cp -v logrotate/searchd /etc/logrotate.d/");
exit;
}
//Liste des configuration
if ($argv[2]=='list') {
echo passthru("tree config/".$hostname."/");
exit;
}
//Configuration
if (!file_exist("config/".$hostname."/".$argv[2])){
echo "Erreur\n"; exit;
}
$server = $argv[2];
$concat = '';
for($i=3;$i<count($argc);$i++){
$concat.= "config/".$hostname."/".$server."/".$argv[$i];
}
exec("cat config/".$hostname."/".$server."/sphinx.conf ".$concact." > /usr/local/sphinx/etc/sphinx.conf");
}