Modification de la configuration et l'installation de sphinx

Suivit mail de michael.
Ce décompose en deux partie l'installation et la configuration des données.

Sphinx.sh => Bash pour l'installation
configuration.php => php pour la creation de la configuration.
This commit is contained in:
Damien LASSERRE 2011-11-08 14:08:29 +00:00
parent 598cc88065
commit 3978dc9328
14 changed files with 185 additions and 316 deletions

103
Sphinx.sh
View File

@ -1,20 +1,15 @@
#!/bin/bash
#This program as been create by Damien for Scores et decisions ste
#Email : dlasserre@scores-decisions.com
# Comatible uniquement sous linux
allVersion=("2.0.1" "0.9.9")
declare -A ERREURS
declare -A ASTUCES
#en cours de developpement !
declare -A ASTUCESDEV
#Color declaration
ROUGE="\\033[1;31m"
NORMAL="\\033[0;39m"
CYAN="\\033[1;36m"
VERT="\\033[1;32m"
BOLD="\\033[1m"
UNDERLINE="\\033[4m"
TEXTNONE="\\033[0m"
@ -26,7 +21,6 @@ dbs="Exist"
sphinx="Exist"
sphinxlog="Exist"
#Tab of erreurs and astuces
ERREURS=(
["CHMOD"]="Execute the script with SUDO or in ROOT"
["DIRECTORY"]="Directory not create"
@ -46,21 +40,18 @@ function loadIndication() {
done
}
# Permet de verifier la valider du superadmin sous linux.
if [ `id -u` = 0 ] #Need Root or Sudo
then
commande=('--configure' '--install' '--all')
commande=('--install' '--all')
for i; do
case $i in
'--help')
echo -e $ROUGE"*************************************************************************************************************************************"$NORMAL
echo -e $ROUGE"*************************************************************************************************************************************"$NORMAL
echo -e "--install version : Vous permet d'installer configurer et crée les indexe et la libstemmer compilation, installation et configuration";
echo -e "--delete : Vous permet de restaurer les parametres de la machines en supprimer la configuration et l'installation"
echo -e "--help : Vous permet d'acceser a l'aide de SphinxInstall"
echo -e "--refresh : Vous permet de rafraichire les indexes, dans le cas d'une modification des fichier source, exemple BDD ou port, mémoir etc..."
echo -e "--version : Vous retournes les dernieres version disponnibles en téléchargement sur Sphinx."
echo -e $ROUGE"**************************************************************************************************************************************"$NORMAL
exit;
echo -e "--delete : Vous permet de restaurer les parametres de la machines en supprimer la configuration et l'installation"
echo -e "--version : Vous retournes les dernieres version disponnibles en téléchargement sur Sphinx."
echo -e $ROUGE"**************************************************************************************************************************************"$NORMAL
exit;
;;
'--delete')
`rm -r indexeConf tmp sphinx-* wget* libste* *~ 2> /dev/null`
@ -72,16 +63,13 @@ then
echo -e "2. Past version : 0.9.9\n"
;;
'--install')
if [ ! -e /dbs ];then `mkdir /dbs`; dbs="Success"; fi
if [ ! -e tmp ];then `mkdir tmp`; fi
if [ ! -e /dbs/sphinx ];then echo -e "/dbs/sphinx Doesn't exist !"; exit; fi
if [ ! -e /dbs/sphinxlog ];then `mkdir /dbs/sphinxlog`; exit; fi
if [ ! -e /usr/local/sphinx ];then `mkdir /dbs/sphinx`; exit; fi
if [ ! -e /init.d/sphinxsearch ];then `cp initscript/sphinxsearch /etc/init.d/ > /dev/null` ;`chmod +x /etc/init.d/sphinxsearch > /dev/null`; `update-rc.d sphinxsearch defaults > /dev/null`; fi
if [ ! -e config/${HOSTNAME} ];then `mkdir config/${HOSTNAME}`;fi
if [ ! -e /usr/local/sphinx ];then `mkdir /dbs/sphinx`; exit; fi
if [ ! -e /init.d/sphinxsearch ];then `cp initscript/sphinxsearch /etc/init.d/ > /dev/null` ;`chmod +x /etc/init.d/sphinxsearch > /dev/null`; `update-rc.d sphinxsearch defaults > /dev/null`; fi
if [ ! -e config/${HOSTNAME} ];then `mkdir config/${HOSTNAME}`;fi
if [ ! -e sphinx-2.0.1.tar.gz ] || [ ! -e sphinx-0.9.9.tar.gz ]
then
if [ ! -z $2 ]; then
@ -129,76 +117,11 @@ then
echo -e $CYAN"Compilation progress..."$ROUGE
if [ $version != "2.0.1-beta" ];then `cd tmp/sphinx-$version ; make; make install ` ;
else
`cd tmp/sphinx-$version/ ; make; make install`;fi
`cd tmp/sphinx-$version/ ; make; make install`;fi
echo -e $NORMAL" - Complited"
echo -n "Machine MASTER ? (y/n) : "; read response
`touch /usr/local/sphinx/etc/sphinx.conf`;
echo -e $ROUGE"Mode MASTER activated"$NORMAL
echo -e "Tapez 'stop' pour quiter l'invite de création des indexes"
while [ indexe != "stop\n" ]; do
echo -e " Entrez indexe : "; read indexe
if [ $response = 'y' ];then
if [ -e config/base/Master/$indexe.conf-MysqlServer ]
then
`cp config/base/Master/$indexe.conf-MysqlServer config/${HOSTNAME}/`
echo -e "Indexe $indexes as been create !";
else
break;
fi
else
if [ -e config/base/Slave/$indexe.conf-MysqlServer ]
then
`cp config/base/Slave/$indexe.conf-MysqlServer config/${HOSTNAME}/`
echo -e "Indexe ''$indexes'' as been create !";
else
break;
fi
fi
done;
`cat config/base/sphinx.conf > /usr/local/sphinx/etc/sphinx.conf `;
`cat config/${HOSTNAME}/*.conf-MysqlServer >> /usr/local/sphinx/etc/sphinx.conf`;
`./Sphinx --refresh`
echo -e $VERT"The installation as been finish ! great ! thank's damien :)"$NORMAL
exit
echo -e $VERT"The installation as been finish ! great ! thank's damien :)"$NORMAL
exit
;;
'--refresh')
`cat config/base/sphinx.conf > /usr/local/sphinx/etc/sphinx.conf`
`cat config/${HOSTNAME}/* >> /usr/local/sphinx/etc/sphinx.conf`
echo -e $VERT"The refresh as been finish ! great ! thank's damien :)"$NORMAL
exit
;;
'--add')
`mkdir config/${HOSTANME}`
echo -n "Machine MASTER ? (y/n) : "; read response
`touch /usr/local/sphinx/etc/sphinx.conf`;
echo -e $ROUGE"Mode MASTER activated"$NORMAL
echo -e "Tapez 'stop' pour quiter l'invite de création des indexes"
while [ indexe != "stop\n" ]; do
echo -e " Entrez indexe : "; read indexe
if [ $response = 'y' ];then
if [ -e config/base/Master/$indexe.conf-MysqlServer ]
then
`cp config/base/Master/$indexe.conf-MysqlServer config/${HOSTNAME}/`
echo -e "Indexe $indexes as been create !";
else
break;
fi
else
if [ -e config/base/Slave/$indexe.conf-MysqlServer ]
then
`cp config/base/Slave/$indexe.conf-MysqlServer config/${HOSTNAME}/`
echo -e "Indexe ''$indexes'' as been create !";
else
break;
fi
fi
done;
`cat config/base/sphinx.conf > /usr/local/sphinx/etc/sphinx.conf `;
`cat config/${HOSTNAME}/*.conf-MysqlServer >> /usr/local/sphinx/etc/sphinx.conf`;
`./Sphinx --refresh`
echo -e $VERT"The installation as been finish ! great ! thank's damien :)"$NORMAL
exit
;;
*)
#Default
echo -e $ROUGE"[Erreur]" ${ERREURS["CMD"]} $NORMAL

View File

@ -38,4 +38,4 @@ index ent
charset_table = 0..9, A..Z->a..z, a..z, \
U+23, U+25, U+26, U+2B, U+3D, U+40, \
U+C0..U+DE->U+E0..U+FE, U+DF, U+E0..U+FF
}
}

View File

@ -28,4 +28,4 @@ index act
charset_table = 0..9, A..Z->a..z, a..z, \
U+23, U+25, U+26, U+2B, U+3D, U+40, \
U+C0..U+DE->U+E0..U+FE, U+DF, U+E0..U+FF
}
}

View File

@ -42,4 +42,4 @@ index dir
charset_table = 0..9, A..Z->a..z, a..z, \
U+23, U+25, U+26, U+2B, U+3D, U+40, \
U+C0..U+DE->U+E0..U+FE, U+DF, U+E0..U+FF
}
}

View File

@ -0,0 +1,77 @@
#############################################################################
## indexer settings
#############################################################################
indexer
{
# memory limit, in bytes, kiloytes (16384K) or megabytes (256M)
# optional, default is 32M, max is 2047M, recommended is 256M to 1024M
mem_limit = 2047M
# maximum IO calls per second (for I/O throttling)
# optional, default is 0 (unlimited)
#
# max_iops = 40
# maximum IO call size, bytes (for I/O throttling)
# optional, default is 0 (unlimited)
#
# max_iosize = 1048576
}
#############################################################################
## searchd settings
#############################################################################
searchd
{
# IP address to bind on
# optional, default is 0.0.0.0 (ie. listen on all interfaces)
#
# address = 127.0.0.1
# address = 192.168.0.1
# searchd TCP port number
# mandatory, default is 3312
listen = 3312
# log file, searchd run info is logged here
# optional, default is 'searchd.log'
log = /dbs/sphinxlog/searchd.log
# query log file, all search queries are logged here
# optional, default is empty (do not log queries)
query_log = /dbs/sphinxlog/query.log
# client read timeout, seconds
# optional, default is 5
read_timeout = 5
# maximum amount of children to fork (concurrent searches to run)
# optional, default is 0 (unlimited)
max_children = 30
# PID file, searchd process ID file name
# mandatory
pid_file = /var/log/searchd.pid
# max amount of matches the daemon ever keeps in RAM, per-index
# WARNING, THERE'S ALSO PER-QUERY LIMIT, SEE SetLimits() API CALL
# default is 1000 (just like Google)
max_matches = 1000
# seamless rotate, prevents rotate stalls if precaching huge datasets
# optional, default is 1
seamless_rotate = 0
# whether to forcibly preopen all indexes on startup
# optional, default is 0 (do not preopen)
preopen_indexes = 0
# whether to unlink .old index copies on succesful rotation.
# optional, default is 1 (do unlink)
unlink_old = 1
}

View File

@ -1,40 +0,0 @@
source comptage
{
type = mysql
sql_host = 192.168.3.30
sql_user = root
sql_pass = scores
sql_db = jo
sql_port = 3306
sql_query = SELECT id,ape_etab,ape_entrep,source,siren,nic,actif,siege,adr_cp,adr_dep,adr_com,(tel>100000000) AS tel,(fax>100000000) AS fax,cj,capital,age_entrep,age_etab,tca,tcaexp,teff_entrep,teff_etab,rang,dateCrea_etab,eff_entrep,eff_etab,nbEtab,bilType,bilAnnee,bilTca,bilCA,bilCAexp,bilRN FROM etablissements_act;
sql_attr_str2ordinal = bilType
sql_attr_uint = source
sql_attr_uint = actif
sql_attr_uint = siege
sql_attr_uint = adr_cp
sql_attr_uint = adr_dep
sql_attr_uint = adr_com
sql_attr_uint = tel
sql_attr_uint = fax
sql_attr_uint = cj
sql_attr_uint = capital
sql_attr_uint = age_entrep
sql_attr_uint = age_etab
sql_attr_uint = tca
sql_attr_uint = tcaexp
sql_attr_uint = teff_entrep
sql_attr_uint = teff_etab
sql_attr_uint = rang
sql_attr_uint = dateCrea_etab
sql_attr_uint = eff_entrep
sql_attr_uint = eff_etab
sql_attr_uint = nbEtab
sql_attr_uint = bilAnnee
sql_attr_uint = bilTca
sql_attr_uint = bilCA
sql_attr_uint = bilCAexp
sql_attr_uint = bilRN
sql_query_info = SELECT siren, nic, ape_etab,ape_entrep, raisonSociale, enseigne, sigle, adr_libVoie, adr_cp, adr_ville FROM etablissements WHERE id=$id;
}

View File

@ -1,45 +0,0 @@
source source_dir
{
type = mysql
sql_host = 192.168.3.30
sql_user = sphinx
sql_pass = indexer
sql_db = jo
sql_query_pre =
sql_query = SELECT id, siren, actif, \
IF(civilite='M',1,IF(civilite='MME' OR civilite='MLLE',2,0)) AS genre, \
CONCAT(nom,' ',naissance_nom, ' ', dirRS) AS nom, prenom, \
YEAR(naissance_date) AS naiss_annee, \
MONTH(naissance_date) AS naiss_mois, \
DAY(naissance_date) AS naiss_jour, \
naissance_lieu, adr_dep \
FROM rncs_dirigeants;
# UNION \
#SELECT id, siren, 0 AS actif, \
#'' AS genre, \
#CONCAT(nom,' ',nomUsage,' ',rs) AS nom, prenom, \
#0 AS naiss_annee, \
#0 AS naiss_mois, \
#0 AS naiss_jour, \
#'' AS naissance_lieu, 0 AS adr_dep \
#FROM bodacc_dirigeants_histo;
sql_attr_uint = naiss_annee
sql_attr_uint = naiss_mois
sql_attr_uint = naiss_jour
sql_attr_uint = actif
sql_attr_uint = genre
sql_attr_uint = adr_dep
}
index dir
{
source = source_dir
path = /dbs/sphinx/dir
docinfo = extern
morphology = none
#ispell_aff = /usr/local/sphinx/etc/francais.aff
charset_type = sbcs
charset_table = 0..9, A..Z->a..z, a..z, \
U+23, U+25, U+26, U+2B, U+3D, U+40, \
U+C0..U+DE->U+E0..U+FE, U+DF, U+E0..U+FF
}

View File

@ -1,23 +0,0 @@
source histo
{
type = mysql
sql_host = 192.168.78.230
sql_user = sphinx_histo
sql_pass = sphinx
sql_db = histobodacc
sql_port = 3306 # optional, default is 3306
sql_query = SELECT id, nomFichier, annee1, dateBod, texte FROM bodacc_ocr;
sql_attr_uint = annee1
sql_query_info = SELECT * FROM bodacc_ocr WHERE id=$id
}
index histo
{
source = histo
path = /home/sphinx/histo
docinfo = extern
morphology = none
min_word_len = 1
charset_type = sbcs
html_strip = 1
}

View File

@ -1,31 +0,0 @@
source source_act
{
type = mysql
sql_host = 192.168.3.30
sql_user = sphinx
sql_pass = indexer
sql_db = jo
sql_query_pre =
sql_query = SELECT id, Siren1, RaisonSociale, Siren2, actif, dateLien, IF(dateUpdate>dateInsert,dateUpdate,dateInsert) as dateMaj, \
source, Pmin*1 AS PminNum, MajMin, PpPm, IF(Pays='','FRA',Pays) AS Pays \
FROM liens \
WHERE ActionPart=1 AND Siren1>1000 AND Siren1 IN (SELECT DISTINCT SIREN FROM etablissements_old);
sql_attr_uint = actif
sql_attr_uint = source
sql_attr_uint = PminNum
sql_attr_str2ordinal= MajMin
sql_attr_str2ordinal= PpPm
sql_attr_str2ordinal= Pays
}
index act
{
source = source_act
path = /dbs/sphinx/act
docinfo = extern
morphology = none
charset_type = sbcs
charset_table = 0..9, A..Z->a..z, a..z, \
U+23, U+25, U+26, U+2B, U+3D, U+40, \
U+C0..U+DE->U+E0..U+FE, U+DF, U+E0..U+FF
}

View File

@ -1,40 +0,0 @@
source comptage
{
type = mysql
sql_host = 192.168.3.30
sql_user = root
sql_pass = scores
sql_db = jo
sql_port = 3306
sql_query = SELECT id,ape_etab,ape_entrep,source,siren,nic,actif,siege,adr_cp,adr_dep,adr_com,(tel>100000000) AS tel,(fax>100000000) AS fax,cj,capital,age_entrep,age_etab,tca,tcaexp,teff_entrep,teff_etab,rang,dateCrea_etab,eff_entrep,eff_etab,nbEtab,bilType,bilAnnee,bilTca,bilCA,bilCAexp,bilRN FROM etablissements_act;
sql_attr_str2ordinal = bilType
sql_attr_uint = source
sql_attr_uint = actif
sql_attr_uint = siege
sql_attr_uint = adr_cp
sql_attr_uint = adr_dep
sql_attr_uint = adr_com
sql_attr_uint = tel
sql_attr_uint = fax
sql_attr_uint = cj
sql_attr_uint = capital
sql_attr_uint = age_entrep
sql_attr_uint = age_etab
sql_attr_uint = tca
sql_attr_uint = tcaexp
sql_attr_uint = teff_entrep
sql_attr_uint = teff_etab
sql_attr_uint = rang
sql_attr_uint = dateCrea_etab
sql_attr_uint = eff_entrep
sql_attr_uint = eff_etab
sql_attr_uint = nbEtab
sql_attr_uint = bilAnnee
sql_attr_uint = bilTca
sql_attr_uint = bilCA
sql_attr_uint = bilCAexp
sql_attr_uint = bilRN
sql_query_info = SELECT siren, nic, ape_etab,ape_entrep, raisonSociale, enseigne, sigle, adr_libVoie, adr_cp, adr_ville FROM etablissements WHERE id=$id;
}

View File

@ -1,44 +0,0 @@
source source_ent
{
type = mysql
sql_host = 192.168.3.30
sql_user = sphinx
sql_pass = indexer
sql_db = jo
sql_query_pre =
sql_query = SELECT id, siren, nic, siege, CONCAT(raisonSociale,\
' "" ',enseigne,' "" ',sigle,' "" ',identite_pre,' "" ',\
REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(sigle,' ',''),\
'@','A'), '&',' ET '),'/',''),',',''),'-',''),'.',''),' ',\
REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(sigle,\
'@','A'), '&',' ET '),'/',''),',',''),'-',''),'.','')) AS nom, adr_dep, \
actif, adr_num, adr_typeVoie, CONCAT(adr_libVoie,' ',adr_comp) AS adresse, adr_cp, \
adr_ville AS ville, cj, ape_etab, (siren>200) AS sirenValide, rang \
FROM etablissements;
sql_attr_str2ordinal = adr_typeVoie
sql_attr_uint = siren
sql_attr_uint = nic
sql_attr_uint = siege
sql_attr_uint = adr_num
sql_attr_uint = adr_cp
sql_attr_uint = adr_dep
sql_attr_uint = actif
sql_attr_uint = cj
sql_attr_uint = sirenValide
sql_attr_uint = rang
}
index ent
{
source = source_ent
path = /dbs/sphinx/ent
docinfo = extern
wordforms = /usr/local/sphinx/etc/wordforms.txt
abreviations = /usr/local/sphinx/etc/abreviations.txt
ispell_aff = /usr/local/sphinx/etc/francais.aff
charset_type = sbcs
charset_table = 0..9, A..Z->a..z, a..z, \
U+23, U+25, U+26, U+2B, U+3D, U+40, \
U+C0..U+DE->U+E0..U+FE, U+DF, U+E0..U+FF
}

92
configure.php Normal file
View File

@ -0,0 +1,92 @@
<?php
Class configure
{
protected $dirname;
protected $sphinx;
protected $hostname;
protected $sql;
protected $indexes;
public function __construct()
{
$this->hostname = exec('echo $(hostname)');
$this->sphinx = 'sphinx.conf';
$this->dirname = 'config/'.$this->hostname.'/';
(file_exists($this->sphinx))?unlink($this->sphinx):'';
}
public function printIndexes($nbServer)
{
$dir = opendir($this->dirname.$this->sql[$nbServer].'/');
$i = 1;
while($file = readdir($dir)) {
if($file != '.' and $file != '..') {
echo "\t\t".'['.$i.'] '. $file."\n";
$this->indexes[$i] = $file;$i++;
}
}
if(count($this->indexes) == 0) {
echo "\tAucun indexes dans le repertoire : '".$this->dirname.$this->sql[$nbServer]."' \n";
exit;
}
}
public function printMysqlServer()
{
$dir = opendir($this->dirname);
$i = 1;
while($rep = readdir($dir)) {
if($rep != '.' and $rep != '..' and is_dir($this->dirname.$rep)) {
echo "\t".'['.$i.'] '.$rep."\n";
$this->sql[$i] = $rep;
$this->printIndexes($i);
$i++;
}
}
if(count($this->sql) == 0) {
echo "\tAucun serveurs dans le repertoire : '".$this->dirname."' \n";
exit;
}
}
public function createConfig($mySqlServer, $index)
{
$dirname = $this->dirname.$mySqlServer.'/';
$dir = opendir($dirname);
if(file_exists($dirname.$index)) {
file_put_contents($this->sphinx, file_get_contents($dirname.$index), FILE_APPEND);
return (true);
}
return (false);
}
public function inviteDeCommande()
{
$handle = fopen('.', 'r');
$server = '';
fwrite(STDOUT, "\n - Serveurs SQL disponnible :\n");
$this->printMysqlServer();
while($server != "exit\n") {
echo "Selectionnez Server et Indexe : ";
$server = fgets(STDIN);
$indexes = explode(' ', $server);
if(isset($this->sql[intval($indexes[0])]) and isset($this->indexes[intval($indexes[1])])) {
$this->createConfig($this->sql[intval($indexes[0])], $this->indexes[intval($indexes[1])]);
echo 'Indexe ajouté !'."\n";
} else
echo 'Cet indexe n\'existe pas !'."\n";
}
file_put_contents($this->sphinx, file_get_contents($this->dirname.$this->sphinx), FILE_APPEND);
exec('sudo mv '.$this->sphinx.' /usr/local/sphinx/etc/'.$this->sphinx);
}
}
$a = new configure();
$a->inviteDeCommande();