What resides in Linux cache? That is the question. How it is related to the docker containers?

Here you will find bunch of the answers; problem solved.

https://hoytech.com/vmtouch/
https://github.com/tobert/pcstat

# pcstat installation
yum install golang.x86_64
go get golang.org/x/sys/unix
go get github.com/tobert/pcstat/pcstat

# vmtouch installtion
git clone https://github.com/hoytech/vmtouch.git
cd vmtouch
make
make install
ln -s /usr/local/bin/vmtouch /sbin/

See how the cache looks like (buff/cache position):

free -g

== Discovery which parts of dockers’ mounts are stored in memory cache ==

docker ps -a  -f status=running --format "{{.Names}}" | \
while read THE_LINE; \
do printf "Docker: %s\n" "$THE_LINE";  \
docker inspect $THE_LINE | jq  -r '.[].Mounts' | grep -o 'Source.*' | sed -r 's|(.*): "(.*)",|\2|g' | xargs -n 1 vmtouch 2>/dev/null  | grep "Resident Pages:"; \
done

== Get detailed information how many of the memory is consumed by processes which use more than 100M of memory. Get cgroup name if the process belongs to it. Show the memory size declared in status file (/proc/$proc_no/status) ==
— in most of cases VmData corresponds to ps -o size —
-s size memory size in kilobytes
VmSize: Virtual memory size.

PIDS=/tmp/cache-info.pids
ps --no-headers  -e -o pid,size | grep -v "\s0$" | sort -nk2 -r | awk '{ if ($2 > 1024*10) {print $1" "$2/1024"mb"}}' | awk '{print $1}' > $PIDS
while read THE_PID; do # get cgroup name
  C_GROUP=$(grep ":memory:" /proc/$THE_PID/cgroup | sed -r 's|(.*memory:/docker/)(.*)|\2|') \
  && DOCKER_NAME=$(docker ps --filter=id=$C_GROUP --format {{.Names}}); SIZE_VM=$(ps --no-header -o size -p $THE_PID);  \
  if [ -z "$C_GROUP" ]; then continue; fi;
  printf "%s %.$2fM pid: %d (cgroup: %s)\n" "$DOCKER_NAME" $(ps --no-headers -fo size -p "$THE_PID" | xargs echo 0.0009765630* | bc) $THE_PID "$C_GROUP"; \
done < $PIDS

== Make an inspection of the container with program pcstat ==

# Inspect container's mounts, searching for files > 10M
DOCKER_NAME="jirastack2"
docker inspect $DOCKER_NAME | jq  -r '.[].Mounts' | grep -o 'Source.*' | sed -r 's|(.*): "(.*)",|\2|g' | xargs -I'{}' find "{}" -size +10M | xargs /root/go/bin/pcstat --plain | sort -nk4

== zabbix minitor with vmtouch (as the sum of cache of the all mounts) ==

# call instructions (it does not wait for the completion and should not been executed simulatnously - hence I used random time of execution)
system.run["sleep $(($RANDOM%1000));sudo docker inspect {HOSTNAME}|jq '.[].Mounts' |grep -o 'Source.*'|sed -r 's|(.*): \"(.*)\",|\2|g'|xargs sudo vmtouch > /tmp/zbx-vmtouch-{HOSTNAME}", nowait]

# read the results
system.run["grep 'R.*s:' /tmp/zbx-vmtouch-{HOSTNAME} |awk {'print $4'} |awk -F'/' {'print $1'} |sed 's/k$/*1024/i'|sed 's/M$/*(1024*1024)/i'|sed 's/G$/*1024^3/i'|bc"]

== Check each container's open files and exam number of cache they are using ==

PIDS=/tmp/cache-info.pids
FILES=/tmp/cache-info.files.
ps --no-headers  -e -o pid,size | grep -v "\s0$" | sort -nk2 -r | awk '{ if ($2 > 1024*10) {print $1" "$2/1024"mb"}}' | awk '{print $1}' > $PIDS
while read THE_PID; do # get cgroup name
  C_GROUP=$(grep ":memory:" /proc/$THE_PID/cgroup | sed -r 's|(.*memory:/docker/)(.*)|\2|') \
  && DOCKER_NAME=$(docker ps --filter=id=$C_GROUP --format {{.Names}}); SIZE_VM=$(ps --no-header -o size -p $THE_PID);   # get docker name\
  lsof -p $THE_PID  2>/dev/null | grep -v "No such file"| awk '{ if ($5=="REG" && $7>10*1048576) { print $ROW } }' > $FILES${DOCKER_NAME}.${THE_PID};
  if [ -n "$C_GROUP" ]; then
    cat /proc/$THE_PID/mountinfo | grep -Ev "localtime|\s/sys|resolv.conf\s|/hostname\s|/hosts\s" | grep -v "^.*\s.*\s0" |  awk '{ if ($5!="/") {print $4 " " $5" "$9}}' | \
    while read THE_LINE; do declare array a=($THE_LINE); H_PREFIX=${a[0]};  C_MOUNT=${a[1]}; H_DEV=${a[2]}; H_MOUNT=$(grep  "^$H_DEV\s" /proc/mounts | awk '{print $2}'); \
    sed -i "s|${C_MOUNT}|${H_MOUNT}${H_PREFIX}|" $FILES${DOCKER_NAME}.${THE_PID} ; \
    done
    # generate the rule to discover files which are on the hosts
  fi
  printf "%d: { 'c-group': '%s', 'docker': '%s', 'ram size mb': '%d Mb'}\n" "$THE_PID" "$C_GROUP" "$DOCKER_NAME" $(($SIZE_VM / 1024));
  FILES_TO_CHECK="";
  while read THE_FILES; do
    FILE_PATH=$(echo "$THE_FILES" | awk '{print $(NF)}')
    if [ -f "$FILE_PATH" ]; then
      FILES_TO_CHECK+=" $FILE_PATH"
    else
      continue;
      printf "unknown: %s\n" "$FILE_PATH";
    fi
  done < $FILES${DOCKER_NAME}.${THE_PID}
  if [ -n "$FILES_TO_CHECK" ]; then
    /root/go/bin/pcstat --plain $FILES_TO_CHECK | sort -nk4;
  fi
done < $PIDS
  1. No comments yet.

  1. No trackbacks yet.