Four years ago I presented the first version of the replication for Sphinx Search. Since that time I have introduced many modifications of the structure: partitioning of delta indices, introducing the distrubuted index. Additionaly, I have changed the system environment from CentOs into LXC (Debian). The storage has been divided into shardes conform to the number of IO, starting from RAM disk, through SSD disk to SATA at the bottom.
\nThe old script has no change to work…, so I rewrite it and <\/p>\n
\r\n#!\/bin\/bash\r\n\r\n# Version 0.1\r\n# since 0.1\r\n# 0.1.1 Support for cold start\r\n# User by start script\r\n# if [ \"$NODE_ROLE\" = \"NODE_ROLE_SLAVE\" ]; then\r\n# \/root\/scripts\/cron\/sphinx-slave.sh\r\n \r\n \r\nSCRIPT_NAME=`basename $0`\r\nMASTER_DIR=\"\/storage_master\/\";\r\nLOCAL_DIR=\"\/storage\/\";\r\nLOG_ERROR_DIR=\"\/var\/log\/sphinx\/slave\/\";\r\nLOG_FILE=\"\/var\/log\/sphinx\/slave.log\";\r\nLOCK_FILE=\"\/tmp\/slave.lock\";\r\nSIGN_FILE=\"\/var\/log\/sphinx\/slave.sgn\";\r\nUNIQUE_NUMBER=\"\";\r\n# glowny plik do monitorowania - powstaje ostatni\r\nFILE_TIMESTAMP=\"sph\"\r\nFILE_TIMESTAMP_QTY=0\r\n# Plik blokady procesu sphinx. Jezeli proces dziala, plik istnieje.\r\nFILE_LOCK_SPH=\"spl\"\r\nMAKE_ROTATE=1;\r\nMAKE_RESTART=0;\r\nCURRENT_UNIX=$((0 + `date +%s`))\r\n\r\nSPH_INDEX=\"\"\r\nSPH_INDEX_DIRS=('daily' 'weekly' 'archive');\r\n# Bazowy indeks, ktory wymaga wylaczenia procesu oraz usuniecia plikow\r\nSPH_DIR_ARCHIVE=\"archive\"\r\nSPH_INDEX_REPORTED=();\r\nSPH_INDEX_REPORTED_ITERATOR=0;\r\nSPH_DIR_WEEKLY=\"weekly\"\r\nSPH_INDEX_WEEKLY_REPORTED=0;\r\nSPH_STOPWORD_FILE=\"stopwords-pl\"\r\n\r\nSPH_INDEX_ERR=();\r\nSPH_INDEX_NEW=();\r\nSPH_INDEX_DIRS_QTY=${#SPH_INDEX_DIRS[@]};\r\nSPH_INDEX_NEW_ITEM_ITERATOR=0;\r\n\r\nFILES_REQUIRED_IN_QTY=8;\r\nFILES_TO_RM=('spl');\r\nFILES_TO_RM_QTY=${#FILES_TO_RM[@]};\r\n\r\nIDX_AWAITING_EXT=\"new\";\r\nIDX_TO_ROTATE=()\r\nIDX_TO_ROTATE_QTY=0;\r\nIDX_TO_ROTATE_ITR=0\r\nIDX_TO_ROTATE_TTL=300\r\nIDX_TO_ROTATE_LOOP_ITR=0\r\n\r\n# The maximum difference between the files (timestamp files) of master and slave\r\n# [s]\r\nMAX_DIFF_INDEX=300\r\n\r\n# regex containing some problems which are not critical\r\n# stale = Stale NFS file handle ERROR\r\nCP_KNOWN_ISSUE=\"Stale\"\r\n\r\nVERBOSE=0\r\n\r\nWS=\"http:\/\/ws-indexer-sph.YOUR_NETWORK\/bootstrap.php?\"\r\n\r\n# To be set\r\nHOMEDIR=\"\";\r\n\r\n# Block functions\r\n\r\nfunction set_homedir {\r\n dirExec=`pwd`;\r\n fileExec=`basename $1`;\r\n fullPath=\"$dirExec\/$fileExec\";\r\n if [ -f $fullPath ]; then\r\n HOMEDIR=$dirExec;\r\n else\r\n HOMEDIR=`dirname $0`;\r\n fi\r\n}\r\n\r\nfunction check_all {\r\n\r\n if [ -f $LOCK_FILE ]; then \r\n\twrite_to_log \"Lock file exists ($LOCK_FILE)\";\r\n\treturn 0; \r\n fi\r\n return 1;\r\n}\r\n\r\nfunction compare_int {\r\n if [ $1 -gt $2 ]; then\r\n\treturn 1;\r\n else\r\n\treturn 0;\r\n fi \r\n}\r\n\r\nfunction write_to_log {\r\n local INFO=\"$1\"\r\n local FOR_VERBOSE_MOD=$2\r\n if [ -z $FOR_VERBOSE_MOD ]; then\r\n\tFOR_VERBOSE_MOD=0\r\n fi\r\n # echo \"$VERBOSE < $FOR_VERBOSE_MOD ?\" >> $LOG_FILE\r\n if [ \"$VERBOSE\" -lt \"$FOR_VERBOSE_MOD\" ]; then\r\n\treturn;\r\n fi\r\n TIME=$(date '+%Y.%m.%d|%H:%M');\r\n echo \"[$UNIQUE_NUMBER] $TIME $INFO\" >> $LOG_FILE\r\n}\r\n\r\nfunction file_update_unix {\r\n\r\nlocal FILE_PATH=\"$1\"\r\nlocal FILE_UNIX_TS=0\r\nif [ -f \"$FILE_PATH\" ]; then\r\n FILE_UNIX_TS=`\/usr\/bin\/stat -c '%Y' \"$FILE_PATH\"`;\r\n FILE_UNIX_TS=$(($FILE_UNIX_TS + 0));\r\nfi\r\necho $FILE_UNIX_TS\r\n}\r\n\r\n\r\nfunction write_to_stderr { \r\n echo \"$@\" 1>&2\r\n}\r\n\r\n# End block functions\r\n\r\n# START\r\nSTART_AT=`date +\"%s.%N\"`\r\n\r\n# Detect homedir\r\nset_homedir $0\r\nUNIQUE_NUMBER=$(date '+%s');\r\n\r\ncheck_all\r\nMAY_CONTINUE=$?;\r\nif [ \"$MAY_CONTINUE\" -eq 0 ]; then\r\n exit 0; \r\nfi\r\n\r\ntouch $LOCK_FILE;\r\ndate > \"$SIGN_FILE\"\r\n\r\nfor ((INDICES_ITERATOR=0;INDICES_ITERATOR<$SPH_INDEX_DIRS_QTY;INDICES_ITERATOR++)); do\r\n SPH_INDEX_DIR=${SPH_INDEX_DIRS[${INDICES_ITERATOR}]};\r\n\r\n # BLOCK Stopword \r\n STOPWORD_MASTER_TS=$(file_update_unix \"$MASTER_DIR$SPH_INDEX_DIR\/$SPH_STOPWORD_FILE\")\r\n write_to_log \"Master stopword ts: $MASTER_DIR$SPH_INDEX_DIR\/$SPH_STOPWORD_FILE : $STOPWORD_MASTER_TS\" 1;\r\n STOPWORD_SLAVE_TS=$(file_update_unix \"$LOCAL_DIR$SPH_INDEX_DIR\/$SPH_STOPWORD_FILE\")\r\n write_to_log \"Slave stopword ts: $LOCAL_DIR$SPH_INDEX_DIR\/$SPH_STOPWORD_FILE : $STOPWORD_SLAVE_TS\" 1;\r\n compare_int $STOPWORD_MASTER_TS $STOPWORD_SLAVE_TS\r\n is_new=$?\r\n write_to_log \"stopword compare_int returns $is_new\" 1;\r\n if [ -z \"$STOPWORD_MASTER_TS\" ]; then\r\n is_new=0\r\n fi\r\n write_to_log \"stopword compare_int returns $is_new\" 1;\r\n if [ \"$is_new\" -eq \"1\" ]; then\r\n cp -p \"$MASTER_DIR$SPH_INDEX_DIR\/$SPH_STOPWORD_FILE\" \"$LOCAL_DIR$SPH_INDEX_DIR\/$SPH_STOPWORD_FILE\" 2>\/tmp\/$SCRIPT_NAME.cp_res.err\r\n cp_result=$?\r\n if [ \"$cp_result\" -gt \"0\" ]; then\r\n\terror_info=$(cat \/tmp\/$SCRIPT_NAME.cp_res.err)\r\n\twrite_to_log \"ERROR at stopwords: $SPH_INDEX_DIR msg: $error_info.\" 0\r\n\twrite_to_stderr \"ERROR at stopwords: $SPH_INDEX_DIR msg: $error_info.\"\r\n else\r\n\twrite_to_log \"Stopwords file '$MASTER_DIR$SPH_INDEX_DIR\/$SPH_STOPWORD_FILE' successfuly copied.\" 0\r\n fi\t\t\r\n \r\n fi\r\n # BLOCK Stopword \r\n FILE_TIMESTAMP_QTY=0 \r\n for SPH_FILE_PATH in $MASTER_DIR$SPH_INDEX_DIR\/*.$FILE_TIMESTAMP ; do\r\n let FILE_TIMESTAMP_QTY++;\r\n # The changes are so fast that sometimes the files disapear\r\n test -f \"$SPH_FILE_PATH\" || continue \r\n # There is temp|new file. Index is builded\"\r\n echo \"$SPH_FILE_PATH\" | grep -c \".new.\" >\/dev\/null && continue\r\n echo \"$SPH_FILE_PATH\" | grep -c \".tmp.\" >\/dev\/null && continue\r\n \r\n SPH_FILE=$(basename \"$SPH_FILE_PATH\")\r\n SPH_INDEX=\"${SPH_FILE%.*}\"\r\n # echo $SPH_INDEX \"($SPH_INDEX_DIR)\"\r\n \r\n if [ -f $MASTER_DIR$SPH_INDEX_DIR\/$SPH_INDEX.$FILE_TIMESTAMP ] && [ -f $MASTER_DIR$SPH_INDEX_DIR\/$SPH_INDEX.$FILE_LOCK_SPH ]; then\r\n\t# Musza istniec obydwa pliki\r\n\tMASTER_STAMP=`\/usr\/bin\/stat -c '%Y' $MASTER_DIR$SPH_INDEX_DIR\/$SPH_INDEX.$FILE_TIMESTAMP`;\r\n MASTER_STAMP=$((MASTER_STAMP + 0));\r\n else\r\n\t# No sph|spl files. spl: master does not work, sph: index does not exists\r\n\tMASTER_STAMP=0;\r\n fi\r\n if [ -f $LOCAL_DIR$SPH_INDEX_DIR\/$SPH_INDEX.$FILE_TIMESTAMP ]; then\r\n\tLOCAL_STAMP=`\/usr\/bin\/stat -c '%Y' $LOCAL_DIR$SPH_INDEX_DIR\/$SPH_INDEX.$FILE_TIMESTAMP`;\r\n\tLOCAL_STAMP=$((LOCAL_STAMP + 0));\r\n else\r\n\tLOCAL_STAMP=0;\r\n fi\r\n \r\n if [ \"$MASTER_STAMP\" -eq 0 ]; then\r\n\twrite_to_log \"Brak indeksu master: $SPH_INDEX ($MASTER_DIR$SPH_INDEX_DIR\/$SPH_INDEX.$FILE_TIMESTAMP)\" 0\r\n else\r\n\t\r\n\tINDEX_DIFF_TS=$(( $MASTER_STAMP - $LOCAL_STAMP))\r\n\tMASTER_DIFF_TS=$(( $CURRENT_UNIX - $MASTER_STAMP))\r\n\t# It can be triggered only when the master index has been built some long time ago (and not before 2 minutes) [second condition]\r\n\tif [ \"$INDEX_DIFF_TS\" -gt \"$MAX_DIFF_INDEX\" ] && [ \"$MASTER_DIFF_TS\" -gt \"$MAX_DIFF_INDEX\" ]; then\r\n\t write_to_log \"Error: index '$SPH_INDEX_DIR\/$SPH_INDEX' is overdue. MASTER|SLAVE: $MASTER_STAMP|$LOCAL_STAMP (Diff=$INDEX_DIFF_TS, Max=$MAX_DIFF_INDEX)\" 0;\t \r\n\t write_to_stderr \"Error: index '$SPH_INDEX_DIR\/$SPH_INDEX' is overdue. MASTER|SLAVE: $MASTER_STAMP|$LOCAL_STAMP (Diff=$INDEX_DIFF_TS, Max=$MAX_DIFF_INDEX)\" \r\n\telse\r\n\t write_to_log \"Index '$SPH_INDEX_DIR\/$SPH_INDEX' is ok. MASTER|SLAVE: $MASTER_STAMP|$LOCAL_STAMP (Diff=$INDEX_DIFF_TS). Master diff=$MASTER_DIFF_TS\" 2;\r\n\tfi\r\n\tcompare_int $MASTER_STAMP $LOCAL_STAMP\r\n\tis_new=$?\r\n\tif [ \"$is_new\" -eq \"1\" ]; then\r\n\t # write logs\r\n\t write_to_log \"Index '$SPH_INDEX_DIR\/$SPH_INDEX' is to be renewing. MASTER|SLAVE: $MASTER_STAMP|$LOCAL_STAMP\" 0;\r\n\t write_to_log \"Stat for index '$SPH_INDEX_DIR\/$SPH_INDEX' MASTER|SLAVE: $MASTER_STAMP|$LOCAL_STAMP\" 1;\t \r\n\t # Sprawdz, czy na pewno nie ma plikow tymczasowych (oznaczajacych, ze indeks jest w trakcie budowy)\r\n\t TMP_FILE_QTY=$(ls -1 -U $MASTER_DIR$SPH_INDEX_DIR\/$SPH_INDEX.*.tmp* 2>\/dev\/null | wc -l);\r\n\t TMP_FILE_QTY=$(( TMP_FILE_QTY + 0));\r\n\t if [ \"$TMP_FILE_QTY\" -gt \"0\" ]; then\r\n\t\twrite_to_log \"Temporary files are available [$TMP_FILE_QTY]. Omitting index renewal - '$SPH_INDEX'.\" 0;\r\n\t\tcontinue;\r\n\t fi\r\n\t \r\n\t SPH_INDEX_COPY_SRC=$MASTER_DIR$SPH_INDEX_DIR\/$SPH_INDEX\".sp?\";\r\n\t SPH_INDEX_COPY_DST=$LOCAL_DIR$SPH_INDEX_DIR\"\/tmp\/\";\r\n\t if [ ! -d \"$SPH_INDEX_COPY_DST\" ]; then\r\n\t\tmkdir \"$SPH_INDEX_COPY_DST\"\r\n\t fi\r\n \t write_to_log \"Start copying: `date +\"%s-%N\"`\" 1\r\n\t write_to_log \"cp -vp $SPH_INDEX_COPY_SRC $SPH_INDEX_COPY_DST | wc -l >\/tmp\/$SCRIPT_NAME.cp_res 2>\/tmp\/$SCRIPT_NAME.cp_res.err\" 2 \t \r\n\t cp -vp $SPH_INDEX_COPY_SRC $SPH_INDEX_COPY_DST 2>\/tmp\/$SCRIPT_NAME.cp_res.err | wc -l >\/tmp\/$SCRIPT_NAME.cp_res 2>>\/tmp\/$SCRIPT_NAME.cp_res.err\r\n\t cp_result=${PIPESTATUS[0]}\r\n \t write_to_log \"Stop copying : `date +\"%s-%N\"`\" 1\r\n \t FILE_QTY=$((0+ `cat \/tmp\/$SCRIPT_NAME.cp_res`))\r\n \t \r\n\t # cp_result=`echo -n $cp_result | sed 's\/\\s\\n\/\/g'`;\r\n\t if [ \"$cp_result\" -gt \"0\" ]; then\r\n\t \tSPH_INDEX_ERR[$INDICES_ITERATOR]=1;\r\n\t\terror_info=$(cat \/tmp\/$SCRIPT_NAME.cp_res.err)\r\n\t\tIS_KNOWN=$(echo \"$error_info\" | grep -c \"$CP_KNOWN_ISSUE\")\r\n\t\twrite_to_log \"ERROR: $SPH_INDEX msg: $error_info. Is known? $IS_KNOWN\" 0\r\n\t\twrite_to_stderr \"Index '$SPH_INDEX' error code: \"$cp_result\", is known? $IS_KNOWN, err-msg: $error_info\"\r\n\t\t# Let sombody know about the problem\r\n\t\tif [ \"$IS_KNOWN\" = \"0\" ]; then\r\n\t\t write_to_log \"Index '$SPH_INDEX' problem during copying the files. (Qty: $FILE_QTY but expected: $FILES_REQUIRED_IN_QTY)\" 0\r\n\t\t write_to_stderr \"Index '$SPH_INDEX' problem during copying the files. Error info: \"$error_info\" (Qty: $FILE_QTY but expected: $FILES_REQUIRED_IN_QTY)\"\r\n\t\tfi\r\n\t\tcontinue;\r\n\t elif [ \"$FILE_QTY\" -ne \"$FILES_REQUIRED_IN_QTY\" ]; then\r\n\t\twrite_to_log \"ERROR: $SPH_INDEX (Unexpected qty of files: $FILE_QTY, expected: $FILES_REQUIRED_IN_QTY)\" 0\t\t\r\n\t\tcontinue;\r\n\t else\r\n\t\twrite_to_log \"Index '$SPH_INDEX' successfuly copied. (Qty: $FILE_QTY expected: $FILES_REQUIRED_IN_QTY)\" 0\r\n\t fi\t\t\r\n \t \r\n\t for ((TO_COPY_ITERATOR=0;TO_COPY_ITERATOR<$FILES_TO_RM_QTY;TO_COPY_ITERATOR++)); do\r\n\t\tFILE_TO_RM=${FILES_TO_RM[${TO_COPY_ITERATOR}]};\r\n \t\twrite_to_log \"Start removing: `date +\"%s-%N\"`\" 2\r\n\t\tcp_result=`rm \"$SPH_INDEX_COPY_DST$SPH_INDEX.$FILE_TO_RM\" 2>&1 `;\r\n \t\twrite_to_log \"Stop removing: `date +\"%s-%N\"`\" 2\r\n\t\tcp_result=`echo -n $cp_result | sed 's\/\\s\\n\/\/g'`;\r\n\t\tif [ -n \"$cp_result\" ]; then\r\n\t \t SPH_INDEX_ERR[$INDICES_ITERATOR]=1;\r\n\t\t MAKE_ROTATE=0;\r\n\t\t write_to_log \"ERROR: $SPH_INDEX $FILE_TO_RM error: $cp_result.\" 0\r\n\t\telse\r\n\t\t write_to_log \"$SPH_INDEX $FILE_TO_RM successfuly removed.\" 2\r\n\t\tfi\t\t\r\n\t done\t \t \r\n\t \r\n\t FILES_REQUIRED_IN_QTY2=$(($FILES_REQUIRED_IN_QTY - $FILES_TO_RM_QTY))\r\n \t FILE_QTY=$(ls -1 -U $SPH_INDEX_COPY_DST$SPH_INDEX.sp? 2>\/dev\/null | \/usr\/bin\/wc -l)\r\n\t FILE_QTY=$((0 + $FILE_QTY))\r\n\t if [ \"$FILE_QTY\" -ne \"$FILES_REQUIRED_IN_QTY2\" ]; then\r\n\t\twrite_to_log \"ERROR: $SPH_INDEX (Unexpected qty of files has been copied: $FILE_QTY, expected: $FILES_REQUIRED_IN_QTY2)\" 0\t\t\r\n\t\tcontinue;\r\n\t else\r\n\t\twrite_to_log \"Notice: index '$SPH_INDEX'. The following qty of files has been copied: $FILE_QTY, expected: $FILES_REQUIRED_IN_QTY2\" 2\r\n\t fi\r\n\t if [ \"$LOCAL_STAMP\" -gt \"0\" ]; then\r\n\t\tcp_result=`rename -f 's\/(.*)\\\/tmp\\\/(.*)\\.\/$1\\\/$2\\.new\\.\/' $SPH_INDEX_COPY_DST$SPH_INDEX.sp? 2>&1 `;\r\n\t else\r\n\t\t# The slave's service is starting\r\n\t\tcp_result=`rename -f 's\/(.*)\\\/tmp\\\/(.*)\\.\/$1\\\/$2\\.\/' $SPH_INDEX_COPY_DST$SPH_INDEX.sp? 2>&1 `;\r\n\t\tMAKE_ROTATE=0;\r\n\t fi\r\n\t cp_result=`echo -n $cp_result | sed 's\/\\s\\n\/\/g'`;\r\n\t if [ -n \"$cp_result\" ]; then\r\n\t \tSPH_INDEX_ERR[$INDICES_ITERATOR]=1;\r\n\t\tMAKE_ROTATE=0;\r\n\t\twrite_to_log \"ERROR: $SPH_INDEX error: $cp_result.\" 0\r\n\t else\r\n\t\twrite_to_log \"$SPH_INDEX successfuly moved.\" 1\r\n\t fi\t\t\r\n\t \r\n\t IS_ERROR=${SPH_INDEX_ERR[${INDICES_ITERATOR}]};\r\n\t if [ -z $IS_ERROR ]; then\r\n\t\tIS_ERROR=0;\r\n\t\t# set index to renew\r\n\t\tSPH_INDEX_NEW[$SPH_INDEX_NEW_ITEM_ITERATOR]=\"$SPH_INDEX_DIR\/$SPH_INDEX\";\r\n\t\t((SPH_INDEX_NEW_ITEM_ITERATOR++));\r\n\t fi\r\n\t if [ \"$LOCAL_STAMP\" -gt \"0\" ]; then\r\n\t\tIDX_TO_ROTATE[${#IDX_TO_ROTATE[*]}]=\"$SPH_INDEX_DIR\/$SPH_INDEX\"\t \r\n\t fi\r\n\t if [ \"$SPH_INDEX_DIR\" = \"$SPH_DIR_ARCHIVE\" ]; then\r\n\t\tSPH_INDEX_REPORTED[$SPH_INDEX_REPORTED_ITERATOR]=\"$SPH_INDEX_DIR\/$SPH_INDEX\";\r\n\t\t((SPH_INDEX_REPORTED_ITERATOR++));\r\n\t fi \r\n\t \r\n\t if [ \"$SPH_INDEX_DIR\" = \"$SPH_DIR_WEEKLY\" ] && [ \"$SPH_INDEX\" = \"9\" ]; then\r\n\t\twrite_to_log \"Index weekly to be reported.\" 0;\r\n\t\tSPH_INDEX_WEEKLY_REPORTED=1\r\n\t fi \r\n\tfi\t\r\n fi\r\n done\r\n if [ \"$FILE_TIMESTAMP_QTY\" -lt \"1\" ]; then\r\n write_to_log \"Qty of filestamps is $FILE_TIMESTAMP_QTY. Probably the master dir is not accessible.\" 0\r\n write_to_stderr \"Qty of filestamps is 0. Probably the master dir is not accessible.\"\r\n else\r\n write_to_log \"Qty of filestamps is $FILE_TIMESTAMP_QTY.\" 0\r\n fi\t\t\r\n\r\ndone \r\n\r\nSPH_INDEX_NEW_QTY=$(( ${#SPH_INDEX_NEW[@]} + 0 ));\r\nSPH_INDEX_ERR_QTY=$(( ${#SPH_INDEX_ERR[@]} + 0 ));\r\n\r\n\r\nif [ \"$SPH_INDEX_NEW_QTY\" -gt \"0\" ] && [ \"$SPH_INDEX_ERR_QTY\" -lt \"1\" ]; then\r\n\r\n write_to_log \"Qty of new indices is larger then 0 and error indices is less then 1 [OK].\" 0;\r\nthen\r\n\tif [ \"$MAKE_ROTATE\" -gt \"0\" ]; then\r\n\t write_to_log \"Executing the command to rotate indices.\" 1\r\n\t command_info=$(\/etc\/init.d\/sphinx rotate 2>&1)\r\n\t retval=$?\r\n\t write_to_log \"$command_info [$retval]\" 0\r\n\t \r\n\t IDX_TO_ROTATE_QTY=${#IDX_TO_ROTATE[@]};\r\n\t # Waiting loop.\r\n\t while [ \"$IDX_TO_ROTATE_LOOP_ITR\" -lt \"$IDX_TO_ROTATE_TTL\" ]; do\r\n\t\tIS_INDEX_AWAITING=0\r\n\t\tfor ((IDX_TO_ROTATE_ITR=0;IDX_TO_ROTATE_ITR<$IDX_TO_ROTATE_QTY;IDX_TO_ROTATE_ITR++)); do\r\n\t\t IDX_AWAITING=${IDX_TO_ROTATE[${IDX_TO_ROTATE_ITR}]};\r\n\t\t test -z $IDX_AWAITING && continue\r\n\t\t # INDEX_ERROR_FILE=\"$LOG_ERROR_DIR$IDX_AWAITING.err\"\t\t\r\n\t\t write_to_log \"Before checking '$IDX_AWAITING' to be rotated.\" 2\r\n\t\t AWAITING_FILES_QTY=$(ls -1 -U $LOCAL_DIR$IDX_AWAITING\".\"$IDX_AWAITING_EXT\".\"* 2>\/dev\/null | \/usr\/bin\/wc -l)\t\t \r\n\t\t AWAITING_FILES_QTY=$((0 + $AWAITING_FILES_QTY))\r\n\t\t if [ \"$AWAITING_FILES_QTY\" -eq \"0\" ]; then\r\n\t\t\tunset IDX_TO_ROTATE[$IDX_TO_ROTATE_ITR]\r\n\t\t else\r\n\t\t\tlet IS_INDEX_AWAITING++;\r\n\t\t fi\r\n\t\t if [ \"$IDX_TO_ROTATE_LOOP_ITR\" -eq \"0\" ]; then\r\n\t\t\twrite_to_log \"Waiting for the index '$IDX_AWAITING' to be rotated.\" 1\r\n\t\t fi\r\n\t\tdone\r\n\t\tlet IDX_TO_ROTATE_LOOP_ITR++;\r\n\t\tif [ \"$IS_INDEX_AWAITING\" -gt \"0\" ]; then\r\n\t\t sleep 2\r\n\t\t \/etc\/init.d\/sphinx status\r\n\t\t res=$?\r\n\t\t if [ \"$res\" -gt \"0\" ]; then\r\n\t\t\twrite_to_log \"ERROR. Cannot rotate - daemon is not working\" 0\r\n\t\t\twrite_to_stderr \"ERROR. Cannot rotate - daemon is not working\"\r\n\t\t\tbreak;\r\n\t\t fi\t\t\r\n\t\telse\r\n\t\t write_to_log \"It looks like the indices have been rotated successfully.\" 0\r\n\t\t break;\r\n\t\tfi\r\n\t\tif [ \"$IDX_TO_ROTATE_LOOP_ITR\" -eq \"$IDX_TO_ROTATE_TTL\" ]; then\r\n\t\t write_to_log \"Reseting iterator to endless loop.\" 0\r\n\t\t write_to_log \"Marking the error occured [$INDEX_ERROR_FILE].\" 0\r\n\t\t # touch $INDEX_ERROR_FILE\r\n\t \t IDX_TO_ROTATE_LOOP_ITR=0\r\n \t\t command_info=$(\/etc\/init.d\/sphinx rotate 2>&1)\r\n\t\t retval=$?\r\n\t\t write_to_log \"$command_info [$retval]\" 0\r\n\t\tfi\r\n\t done\t \t \r\n\t # wait till rotation is executed\r\n\telse\r\n\t write_to_log \"Not allowed to rotate indices\" 0;\r\n\tfi\r\n # fi\r\n \r\nelse\r\n write_to_log \"Qty of new indices [$SPH_INDEX_NEW_QTY] is less then 0 OR error indices [$SPH_INDEX_ERR_QTY] is greater then 0.\"; \r\nfi \r\n\r\nSPH_INDEX_REPORTED_QTY=$(( ${#SPH_INDEX_REPORTED[@]} + 0 ));\r\nif [ \"$SPH_INDEX_REPORTED_QTY\" -gt \"0\" ]; then\r\n write_to_log \"Qty of index to report -gt 0 [OK].\" 0;\r\n for ((IDX_TO_ROTATE_ITR=0;IDX_TO_ROTATE_ITR<$SPH_INDEX_REPORTED_QTY;IDX_TO_ROTATE_ITR++)); do\r\n\tIDX_AWAITING=${SPH_INDEX_REPORTED[${IDX_TO_ROTATE_ITR}]};\r\n\t\t\r\n\r\n\tVERSION_IDX=$(stat -c '%Y' \"$LOCAL_DIR$IDX_AWAITING.sph\")\r\n\tSPH_INDEX=`basename $IDX_AWAITING`\r\n\tSPH_INDEX_DIR=`dirname $IDX_AWAITING | xargs basename`\r\n\t\r\n\tPHYSICAL_INDEX=$SPH_INDEX_DIR\"_p_\"$SPH_INDEX\r\n\twrite_to_log \"Physical index: $PHYSICAL_INDEX version: $VERSION_IDX\" 0\r\n\tWS_URL=$WS\"t=3&version=$VERSION_IDX&physical_index=$PHYSICAL_INDEX&i=$SPH_INDEX_DIR\"\r\n\t\/usr\/bin\/wget -q -t 1 $WGET_OPTIONS $WS_URL --output-document=\"\/tmp\/master_set_idx_$SPH_INDEX$SPH_INDEX_DIR\" 2>\/tmp\/on_delta_built_err\r\n\tresult=$?\r\n\tif [ \"$result\" -gt \"0\" ]; then\r\n\t err_info=`cat > \"\/tmp\/on_delta_built_err\"`\r\n\t write_to_stderr \"Error on $WS_URL. Content: $err_info\"\r\n\tfi\r\n\tdata_tmp=$(cat \"\/tmp\/master_set_idx_$SPH_INDEX$SPH_INDEX_DIR\")\r\n\twrite_to_log \"$data_tmp\" 0\r\n done\r\nfi\r\n\r\nif [ \"$SPH_INDEX_WEEKLY_REPORTED\" -eq 1 ]; then\r\n\twrite_to_log \"Qty of index weekly to report -eq 1 [OK].\" 0\r\n\twrite_to_log \"File: $LOCAL_DIR\"\"weekly9.sph.\" 0\r\n\tVERSION_IDX=$(stat -c '%Y' \"$LOCAL_DIR$SPH_DIR_WEEKLY\/9.sph\")\r\n\twrite_to_log \"Index weekly in version: $VERSION_IDX\" 0\r\n\tWS_URL=$WS\"t=COMMAND_ON_DELTA_BUILT&i=weekly&version=$VERSION_IDX\"\r\n\t\/usr\/bin\/wget -q -t 1 $WGET_OPTIONS $WS_URL --output-document=\"\/tmp\/on_delta_built\" 2>\/tmp\/on_delta_built_err\r\n\tresult=$?\r\n\tif [ \"$result\" -gt \"0\" ]; then\r\n\t err_info=`cat > \"\/tmp\/on_delta_built_err\"`\r\n\t write_to_stderr \"Error on $WS_URL. Content: $err_info\"\r\n\tfi\r\n\tdata_tmp=$(cat \"\/tmp\/on_delta_built\")\r\n\twrite_to_log \"$data_tmp\" 0\r\nfi\r\n\r\n\r\nrm $LOCK_FILE;\r\n\r\nSTOP_AT=`date +\"%s.%N\"`\r\n\r\n\r\nDURATION=` echo \"$STOP_AT - $START_AT\" | bc ` \r\nwrite_to_log \"End of work $DURATION [s]\" 0;\r\nMEM_INFO=`grep \"^MemFree\\|^Cached\\|SwapCached\\|Inactive:\\|SwapFree\" \/proc\/meminfo`\r\nwrite_to_log \"$MEM_INFO\" 0;\r\n\r\nexit 0\r\n<\/pre>\n","protected":false},"excerpt":{"rendered":"Four years ago I presented the first version of the replication for Sphinx Search. Since that time I have introduced many modifications of the structure: partitioning of delta indices, introducing the distrubuted index. Additionaly, I have changed the system environment from CentOs into LXC (Debian). The storage has been divided into shardes conform to the […]<\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":[],"categories":[1],"tags":[],"_links":{"self":[{"href":"http:\/\/10sa.com\/sql_stories\/index.php?rest_route=\/wp\/v2\/posts\/807"}],"collection":[{"href":"http:\/\/10sa.com\/sql_stories\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"http:\/\/10sa.com\/sql_stories\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"http:\/\/10sa.com\/sql_stories\/index.php?rest_route=\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"http:\/\/10sa.com\/sql_stories\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=807"}],"version-history":[{"count":5,"href":"http:\/\/10sa.com\/sql_stories\/index.php?rest_route=\/wp\/v2\/posts\/807\/revisions"}],"predecessor-version":[{"id":812,"href":"http:\/\/10sa.com\/sql_stories\/index.php?rest_route=\/wp\/v2\/posts\/807\/revisions\/812"}],"wp:attachment":[{"href":"http:\/\/10sa.com\/sql_stories\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=807"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"http:\/\/10sa.com\/sql_stories\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=807"},{"taxonomy":"post_tag","embeddable":true,"href":"http:\/\/10sa.com\/sql_stories\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=807"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}