My Sphinx Search replication (master and slaves)
One instance and one storage with indices is neither secure nor effective. That is why I created my replication for Sphinx Search . To put it shortly, my Sphinx’s architecture constists of tree indices and MVA (multivalue attributes stored in mva_updates_pool) – that is why I cannot use RT index.
My first idea was to create independent Sphix index + searchd(aemon) per machine which would be self-sufficient – but it has tree disadvantagies:
– there is one source (i.e. database :));
– advanced structure in database (as manager function);
– does not conform to KISS;
Therefore, a known solution from ancient times – one master many slaves – seems to be better.
Master works as usual and its additional requirement is NFS server ON. You should configure your iptables – this is my shot (considering that slave is 192.168.1.51):
ACCEPT tcp -- 192.168.1.51 0.0.0.0/0 tcp dpt:32803
ACCEPT udp -- 192.168.1.51 0.0.0.0/0 udp dpt:32769
ACCEPT tcp -- 192.168.1.51 0.0.0.0/0 tcp dpt:2049
ACCEPT udp -- 192.168.1.51 0.0.0.0/0 udp dpt:2049
ACCEPT tcp -- 192.168.1.51 0.0.0.0/0 tcp dpt:892
ACCEPT udp -- 192.168.1.51 0.0.0.0/0 udp dpt:892
ACCEPT tcp -- 192.168.1.51 0.0.0.0/0 tcp dpt:875
ACCEPT udp -- 192.168.1.51 0.0.0.0/0 udp dpt:875
ACCEPT udp -- 192.168.1.51 0.0.0.0/0 udp dpt:111
ACCEPT tcp -- 192.168.1.51 0.0.0.0/0 tcp dpt:111
Share master storage giving adequate writing to /etc/exports
/storage/ 192.168.1.51(rw,no_subtree_check,sync,no_root_squash)
Now slave, firstly you should mount master through nfs in yout fstab:
# 192.168.1.50 is master
192.168.1.50:/storage /storage_master nfs ro,rsize=8192,hard,intr,nfsvers=3,tcp,noatime,nodev,async 0 0
Ok, slave sees master storage, now I would like to explain how my script works:
1. Script observes whether master’s index change;
2. In case master’s index has been rebuilt it copies all files conform to sphinx rotate requirements (conforming to Sphinx sources);
3. It executes special stop/start file with option rotate (I present this file as /etc/init.d/sphinx)
/etc/init.d/sphinx
How to install
On redhat/centos Os you execute:
chkconfig --add sphinx
Content
#!/bin/bash
# inits, open order, close order
# chkconfig: 2345 99 10
# description: Sphinx daemon
HOME_DIR="/usr/local/services/sphinx/bin/"
STORAGE_DIR="/storage"
NAME="searchd"
CONFIG="/usr/local/services/sphinx/etc/arch.cnf"
iterator=2
MAX_ATTEMPT=5
aINDEXES=(daily weekly archive)
INDEX_PID_EXT="spl"
if [ ! -r $CONFIG ]; then
echo "No config file or cannot be read. Script has been stopped"
exit 0
fi
if [ ! -d $HOME_DIR ]; then
echo "No home dir. Script has been stopped"
exit 0
fi
# Function unlinks files containg Sphinxa indices' pids (they exists when daemon is ugly killed)
del_index_pid(){
element_count=${#aINDEXES[@]}
index=0
while [ "$index" -lt "$element_count" ]
do
INDEX_FILE=${aINDEXES[$index]}
INDEX_FILE_PATH="$STORAGE_DIR/$INDEX_FILE.$INDEX_PID_EXT"
if [ -f "$INDEX_FILE_PATH" ]; then
rm $INDEX_FILE_PATH
if [ -f "$INDEX_FILE_PATH" ]; then
"Error: cannot remove PID index file in localization: $INDEX_FILE_PATH"
else
"Warning: Sphinx was not clearly closed. I removed index PID file: $INDEX_FILE_PATH"
fi
# echo "$INDEX_FILE_PATH does not exists."
fi
index+=1
done
}
check_pid(){
DAEMON_PID=$(/sbin/pidof -s -o %PPID "$HOME_DIR$NAME")
DAEMON_PID=$(($DAEMON_PID+0))
usleep 500000
}
start_process() {
if [ $DAEMON_PID -gt 1 ]; then
echo "Process is running ($DAEMON_PID)"
else
del_index_pid
echo "Starting..."
echo "$HOME_DIR$NAME" --config "$CONFIG"
"$HOME_DIR$NAME" --config "$CONFIG"
rebuild;
RETVAL=$?
fi
}
kill_process() {
if [ $DAEMON_PID -lt 1 ]; then
echo "Process is not running $DAEMON_PID"
else
if [ $iterator -gt 1 ]; then
echo "Killing $NAME..."
fi
killall $NAME
sleep 1
check_pid
if [ ! $DAEMON_PID ]; then
echo "Process killed $DAEMON_PID"
else
if [ $iterator -gt $MAX_ATTEMPT ]; then
echo "I cannot kill process. I give up after $MAX_ATTEMPT attempts. Sorry, I fucked it."
else
iterator=$(( $iterator + 1 ))
kill_process
fi
fi
fi
}
function rotate {
if [ $DAEMON_PID -lt 1 ]; then
echo "Process is not running $DAEMON_PID"
else
echo "Process is to be rotated"
kill -HUP $DAEMON_PID;
fi
}
check_status() {
if [ $DAEMON_PID -gt 1 ]; then
echo "1"
else
usleep 500000
check_pid
check_status2
fi
}
check_status2() {
if [ $DAEMON_PID -gt 1 ]; then
echo "1"
else
echo "-1"
fi
}
check_pid
case "$1" in
start)
start_process
;;
stop)
kill_process
;;
restart)
kill_process
start_process
;;
status)
check_status
;;
rotate)
rotate
;;
*)
echo "Usage: $0 (start|stop|restart|status|rotate)"
;;
esac
Ok, now the main script self-descripting “sphinx-slave.sh”
#!/bin/bash
MASTER_DIR="/storage_master/arch/";
LOCAL_DIR="/storage/arch/";
LOG_ERROR_DIR="/var/log/sphinx/slave/";
LOG_FILE="/var/log/sphinx/slave.log";
LOCK_FILE="/var/log/sphinx/slave.lock";
UNIQUE_NUMBER="";
# basic file to watch - it is created as last.
FILE_TIMESTAMP="sph"
# Lock file of Sphinx process. File is present while process is working.
FILE_LOCK_SPH="spl"
MAKE_ROTATE=1;
MAKE_RESTART=0;
SPH_INDICES=('daily' 'weekly' 'archive');
# Basic index which requires to stop searchd(aemon).
SPH_MAIN_INDEX="archive";
SPH_INDEX_ERR=();
SPH_INDEX_NEW=();
SPH_INDICES_QTY=${#SPH_INDICES[@]};
SPH_INDEX_NEW_ITEM_ITERATOR=0;
FILES_TO_DETECT=('spd' 'sph' 'spi' 'spp');
FILES_TO_DETECT_QTY=${#FILES_TO_DETECT[@]};
FILES_TO_RENAME=('spl' 'spa' 'spd' 'sph' 'spi' 'spk' 'spm' 'spp');
FILES_TO_RENAME_QTY=${#FILES_TO_RENAME[@]};
# To be set
HOMEDIR="";
# Block functions
function set_homedir {
dirExec=`pwd`;
fileExec=`basename $1`;
fullPath="$dirExec/$fileExec";
if [ -f $fullPath ]; then
HOMEDIR=$dirExec;
else
HOMEDIR=`dirname $0`;
fi
}
function check_all {
if [ -f $LOCK_FILE ]; then
write_to_log "Lock file exists ($LOCK_FILE)";
return 0;
fi
return 1;
}
function compare_int {
if [ $1 -gt $2 ]; then
return 1;
else
return 0;
fi
}
function write_to_log {
INFO=$1
TIME=$(date '+%Y.%m.%d|%H:%M');
echo "[$UNIQUE_NUMBER] $TIME $INFO" >> $LOG_FILE
}
# End block functions
# Detect homedir
set_homedir $0
UNIQUE_NUMBER=$(date '+%s');
check_all
MAY_CONTINUE=$?;
if [ "$MAY_CONTINUE" -eq 0 ]; then
exit 0;
fi
touch $LOCK_FILE;
for ((INDICES_ITERATOR=0;INDICES_ITERATOR<$SPH_INDICES_QTY;INDICES_ITERATOR++)); do SPH_INDEX=${SPH_INDICES[${INDICES_ITERATOR}]}; if [ -f $MASTER_DIR$SPH_INDEX.$FILE_TIMESTAMP ] && [ -f $MASTER_DIR$SPH_INDEX.$FILE_LOCK_SPH ]; then # Musza istniec obydwa pliki MASTER_STAMP=`/usr/bin/stat -c '%Z' $MASTER_DIR$SPH_INDEX.$FILE_TIMESTAMP`; MASTER_STAMP=$((MASTER_STAMP + 0)); else # No sph|spl files. spl: master does not work, sph: index does not exists MASTER_STAMP=0; fi if [ -f $LOCAL_DIR$SPH_INDEX.$FILE_TIMESTAMP ]; then LOCAL_STAMP=`/usr/bin/stat -c '%Z' $LOCAL_DIR$SPH_INDEX.$FILE_TIMESTAMP`; LOCAL_STAMP=$((LOCAL_STAMP + 0)); else LOCAL_STAMP=0; fi if [ "$MASTER_STAMP" -eq 0 ]; then write_to_log "Brak indeksu master: $SPH_INDEX" else INDEX_ERROR_FILE="$LOG_ERROR_DIR$SPH_INDEX.err"; # echo "$MASTER_STAMP for $SPH_INDEX"; compare_int $MASTER_STAMP $LOCAL_STAMP is_new=$? if [ "$is_new" -eq "1" ]; then if [ -f $INDEX_ERROR_FILE ]; then rm $INDEX_ERROR_FILE; fi # write logs write_to_log "Index '$SPH_INDEX' is to be renewing"; write_to_log "Stat for index '$SPH_INDEX' MASTER|SLAVE: $MASTER_STAMP|$LOCAL_STAMP"; # Sprawdz, czy na pewno nie ma plikow tymczasowych (oznaczajacych, ze indeks jest w trakcie budowy) TMP_FILE_QTY=$(ls -l $MASTER_DIR$SPH_INDEX.*.tmp* 2>/dev/null | wc -l);
TMP_FILE_QTY=$(( TMP_FILE_QTY + 0));
if [ "$TMP_FILE_QTY" -gt "0" ]; then
write_to_log "Temporary files are available [$TMP_FILE_QTY]. Omitting index renewal - '$SPH_INDEX'.";
continue;
fi
if [ "$SPH_MAIN_INDEX" = "$SPH_INDEX" ]; then
write_to_log "Stopping service"
write_to_log "Copying file with stopwords"
cp $MASTER_DIR../stopwords-pl $LOCAL_DIR../
/etc/init.d/sphinx stop
write_to_log "Executing command 'sync'";
sync;
write_to_log "Writing command to clear cache";
echo 3 > /proc/sys/vm/drop_caches;
sleep 10;
fi
# copy files as .new
for ((TO_COPY_ITERATOR=0;TO_COPY_ITERATOR<$FILES_TO_RENAME_QTY;TO_COPY_ITERATOR++)); do SPH_INDEX_COPY_SRC=$MASTER_DIR$SPH_INDEX"."; SPH_INDEX_COPY_DST=$LOCAL_DIR$SPH_INDEX"."; FILE_TO_COPY=${FILES_TO_RENAME[${TO_COPY_ITERATOR}]}; # echo $SPH_INDEX_COPY$FILE_TO_COPY; if [ "$SPH_MAIN_INDEX" = "$SPH_INDEX" ]; then rm_result=`rm -f $SPH_INDEX_COPY_DST$FILE_TO_COPY 2>&1`;
res_int=$(( $? + 0 ));
rm_result=`echo -n $rm_result | sed 's/\s\n//g'`;
rm_no_file=`echo $rm_result | grep -i "No such file"`;
# the length of STRING is zero
if [ $res_int -gt "0" ] && [ -z "$rm_no_file" ]; then
echo "Error: -z rm_nofile: $rm_no_file vs. result: $rm_result vs. $SPH_INDEX_COPY_DST$FILE_TO_COPY"
SPH_INDEX_ERR[$INDICES_ITERATOR]=1;
write_to_log "$SPH_INDEX $rm_result";
fi
cp_result=`cp $SPH_INDEX_COPY_SRC$FILE_TO_COPY $SPH_INDEX_COPY_DST$FILE_TO_COPY 2>&1 `;
else
# add extenstion ".new"
cp_result=`cp $SPH_INDEX_COPY_SRC$FILE_TO_COPY $SPH_INDEX_COPY_DST"new."$FILE_TO_COPY 2>&1 `;
fi
cp_result=`echo -n $cp_result | sed 's/\s\n//g'`;
# echo $cp_result;
if [ -n "$cp_result" ]; then
SPH_INDEX_ERR[$INDICES_ITERATOR]=1;
MAKE_ROTATE=0;
write_to_log "$SPH_INDEX $FILE_TO_COPY error: $cp_result."
else
write_to_log "$SPH_INDEX $FILE_TO_COPY successfuly copied."
fi
done
IS_ERROR=${SPH_INDEX_ERR[${INDICES_ITERATOR}]};
if [ -z $IS_ERROR ]; then
IS_ERROR=0;
# set index to renew
SPH_INDEX_NEW[$SPH_INDEX_NEW_ITEM_ITERATOR]=$SPH_INDEX;
((SPH_INDEX_NEW_ITEM_ITERATOR++));
fi
if [ "$IS_ERROR" -eq "1" ] ; then
touch $INDEX_ERROR_FILE;
fi
fi
fi
done
SPH_INDEX_NEW_QTY=$(( ${#SPH_INDEX_NEW[@]} + 0 ));
SPH_INDEX_ERR_QTY=$(( ${#SPH_INDEX_ERR[@]} + 0 ));
if [ "$SPH_INDEX_NEW_QTY" -gt "0" ] && [ "$SPH_INDEX_ERR_QTY" -lt "1" ]; then
write_to_log "Qty of new indices is larger then 0 and error indices is less then 1 [OK].";
for ((INDICES_ITERATOR2=0;INDICES_ITERATOR2<$SPH_INDEX_NEW_QTY;INDICES_ITERATOR2++)); do
SPH_INDEX_NAME=${SPH_INDEX_NEW[${INDICES_ITERATOR2}]};
if [ "$SPH_INDEX_NAME" = $SPH_MAIN_INDEX ]; then
MAKE_RESTART=1;
else
continue;
# for IDX_FILE_TMP in "$LOCAL_DIR$SPH_INDEX_NAME"*".new"; do
# IDX_FILE=`echo $IDX_FILE_TMP | sed 's/\.tmp$//g'`;
# mv $IDX_FILE_TMP $IDX_FILE;
# done
fi
done
if [ "$MAKE_RESTART" -gt "0" ]; then
write_to_log "Starting service"
/etc/init.d/sphinx start
else
if [ "$MAKE_ROTATE" -gt "0" ]; then
write_to_log "Rotating indexes"
/etc/init.d/sphinx rotate
else
write_to_log "Not allowed to rotate indices";
fi
fi
# else
# write_to_log "Qty of new indices [$SPH_INDEX_NEW_QTY] is less then 0 OR error indices [$SPH_INDEX_ERR_QTY] is greater then 0.";
fi
rm $LOCK_FILE;
exit 0
No comments yet.