Here you will find bunch of the answers; problem solved.<\/p>\n
https:\/\/hoytech.com\/vmtouch\/<\/a> See how the cache looks like (buff\/cache position):<\/p>\n == Discovery which parts of dockers’ mounts are stored in memory cache ==<\/p>\n == Get detailed information how many of the memory is consumed by processes which use more than 100M of memory. Get cgroup name if the process belongs to it. Show the memory size declared in status file (\/proc\/$proc_no\/status) == == Make an inspection of the container with program pcstat ==<\/p>\n == zabbix minitor with vmtouch (as the sum of cache of the all mounts) ==<\/p>\n == Check each container's open files and exam number of cache they are using ==<\/p>\n Here you will find bunch of the answers; problem solved. https:\/\/hoytech.com\/vmtouch\/ https:\/\/github.com\/tobert\/pcstat # pcstat installation yum install golang.x86_64 go get golang.org\/x\/sys\/unix go get github.com\/tobert\/pcstat\/pcstat # vmtouch installtion git clone https:\/\/github.com\/hoytech\/vmtouch.git cd vmtouch make make install ln -s \/usr\/local\/bin\/vmtouch \/sbin\/ See how the cache looks like (buff\/cache position): free -g == Discovery which parts of dockers’ […]<\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":[],"categories":[19],"tags":[],"_links":{"self":[{"href":"http:\/\/10sa.com\/sql_stories\/index.php?rest_route=\/wp\/v2\/posts\/1217"}],"collection":[{"href":"http:\/\/10sa.com\/sql_stories\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"http:\/\/10sa.com\/sql_stories\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"http:\/\/10sa.com\/sql_stories\/index.php?rest_route=\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"http:\/\/10sa.com\/sql_stories\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=1217"}],"version-history":[{"count":23,"href":"http:\/\/10sa.com\/sql_stories\/index.php?rest_route=\/wp\/v2\/posts\/1217\/revisions"}],"predecessor-version":[{"id":1240,"href":"http:\/\/10sa.com\/sql_stories\/index.php?rest_route=\/wp\/v2\/posts\/1217\/revisions\/1240"}],"wp:attachment":[{"href":"http:\/\/10sa.com\/sql_stories\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=1217"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"http:\/\/10sa.com\/sql_stories\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=1217"},{"taxonomy":"post_tag","embeddable":true,"href":"http:\/\/10sa.com\/sql_stories\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=1217"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}
\nhttps:\/\/github.com\/tobert\/pcstat<\/a><\/p>\n\r\n# pcstat installation\r\nyum install golang.x86_64\r\ngo get golang.org\/x\/sys\/unix\r\ngo get github.com\/tobert\/pcstat\/pcstat\r\n\r\n# vmtouch installtion\r\ngit clone https:\/\/github.com\/hoytech\/vmtouch.git\r\ncd vmtouch\r\nmake\r\nmake install\r\nln -s \/usr\/local\/bin\/vmtouch \/sbin\/\r\n<\/pre>\n
\r\nfree -g\r\n<\/pre>\n
\r\ndocker ps -a -f status=running --format \"{{.Names}}\" | \\\r\nwhile read THE_LINE; \\\r\ndo printf \"Docker: %s\\n\" \"$THE_LINE\"; \\\r\ndocker inspect $THE_LINE | jq -r '.[].Mounts' | grep -o 'Source.*' | sed -r 's|(.*): \"(.*)\",|\\2|g' | xargs -n 1 vmtouch 2>\/dev\/null | grep \"Resident Pages:\"; \\\r\ndone\r\n<\/pre>\n
\n— in most of cases VmData corresponds to ps -o size —
\n-s size memory size in kilobytes
\nVmSize: Virtual memory size.<\/p>\n\r\nPIDS=\/tmp\/cache-info.pids\r\nps --no-headers -e -o pid,size | grep -v \"\\s0$\" | sort -nk2 -r | awk '{ if ($2 > 1024*10) {print $1\" \"$2\/1024\"mb\"}}' | awk '{print $1}' > $PIDS\r\nwhile read THE_PID; do # get cgroup name\r\n C_GROUP=$(grep \":memory:\" \/proc\/$THE_PID\/cgroup | sed -r 's|(.*memory:\/docker\/)(.*)|\\2|') \\\r\n && DOCKER_NAME=$(docker ps --filter=id=$C_GROUP --format {{.Names}}); SIZE_VM=$(ps --no-header -o size -p $THE_PID); \\\r\n if [ -z \"$C_GROUP\" ]; then continue; fi;\r\n printf \"%s %.$2fM pid: %d (cgroup: %s)\\n\" \"$DOCKER_NAME\" $(ps --no-headers -fo size -p \"$THE_PID\" | xargs echo 0.0009765630* | bc) $THE_PID \"$C_GROUP\"; \\\r\ndone < $PIDS\r\n<\/pre>\n\r\n# Inspect container's mounts, searching for files > 10M\r\nDOCKER_NAME=\"jirastack2\"\r\ndocker inspect $DOCKER_NAME | jq -r '.[].Mounts' | grep -o 'Source.*' | sed -r 's|(.*): \"(.*)\",|\\2|g' | xargs -I'{}' find \"{}\" -size +10M | xargs \/root\/go\/bin\/pcstat --plain | sort -nk4\r\n<\/pre>\n\r\n# call instructions (it does not wait for the completion and should not been executed simulatnously - hence I used random time of execution)\r\nsystem.run[\"sleep $(($RANDOM%1000));sudo docker inspect {HOSTNAME}|jq '.[].Mounts' |grep -o 'Source.*'|sed -r 's|(.*): \\\"(.*)\\\",|\\2|g'|xargs sudo vmtouch > \/tmp\/zbx-vmtouch-{HOSTNAME}\", nowait]\r\n\r\n# read the results\r\nsystem.run[\"grep 'R.*s:' \/tmp\/zbx-vmtouch-{HOSTNAME} |awk {'print $4'} |awk -F'\/' {'print $1'} |sed 's\/k$\/*1024\/i'|sed 's\/M$\/*(1024*1024)\/i'|sed 's\/G$\/*1024^3\/i'|bc\"]\r\n<\/pre>\n\r\nPIDS=\/tmp\/cache-info.pids\r\nFILES=\/tmp\/cache-info.files.\r\nps --no-headers -e -o pid,size | grep -v \"\\s0$\" | sort -nk2 -r | awk '{ if ($2 > 1024*10) {print $1\" \"$2\/1024\"mb\"}}' | awk '{print $1}' > $PIDS\r\nwhile read THE_PID; do # get cgroup name\r\n C_GROUP=$(grep \":memory:\" \/proc\/$THE_PID\/cgroup | sed -r 's|(.*memory:\/docker\/)(.*)|\\2|') \\\r\n && DOCKER_NAME=$(docker ps --filter=id=$C_GROUP --format {{.Names}}); SIZE_VM=$(ps --no-header -o size -p $THE_PID); # get docker name\\\r\n lsof -p $THE_PID 2>\/dev\/null | grep -v \"No such file\"| awk '{ if ($5==\"REG\" && $7>10*1048576) { print $ROW } }' > $FILES${DOCKER_NAME}.${THE_PID};\r\n if [ -n \"$C_GROUP\" ]; then\r\n cat \/proc\/$THE_PID\/mountinfo | grep -Ev \"localtime|\\s\/sys|resolv.conf\\s|\/hostname\\s|\/hosts\\s\" | grep -v \"^.*\\s.*\\s0\" | awk '{ if ($5!=\"\/\") {print $4 \" \" $5\" \"$9}}' | \\\r\n while read THE_LINE; do declare array a=($THE_LINE); H_PREFIX=${a[0]}; C_MOUNT=${a[1]}; H_DEV=${a[2]}; H_MOUNT=$(grep \"^$H_DEV\\s\" \/proc\/mounts | awk '{print $2}'); \\\r\n sed -i \"s|${C_MOUNT}|${H_MOUNT}${H_PREFIX}|\" $FILES${DOCKER_NAME}.${THE_PID} ; \\\r\n done\r\n # generate the rule to discover files which are on the hosts\r\n fi\r\n printf \"%d: { 'c-group': '%s', 'docker': '%s', 'ram size mb': '%d Mb'}\\n\" \"$THE_PID\" \"$C_GROUP\" \"$DOCKER_NAME\" $(($SIZE_VM \/ 1024));\r\n FILES_TO_CHECK=\"\";\r\n while read THE_FILES; do\r\n FILE_PATH=$(echo \"$THE_FILES\" | awk '{print $(NF)}')\r\n if [ -f \"$FILE_PATH\" ]; then\r\n FILES_TO_CHECK+=\" $FILE_PATH\"\r\n else\r\n continue;\r\n printf \"unknown: %s\\n\" \"$FILE_PATH\";\r\n fi\r\n done < $FILES${DOCKER_NAME}.${THE_PID}\r\n if [ -n \"$FILES_TO_CHECK\" ]; then\r\n \/root\/go\/bin\/pcstat --plain $FILES_TO_CHECK | sort -nk4;\r\n fi\r\ndone < $PIDS\r\n<\/pre>\n","protected":false},"excerpt":{"rendered":"