#!/bin/bash

#for debugging
#set -x

#global enable/disable
GX_LOWMEM_HELPER_ENABLE=false

#enable/disable debug logging
GX_LOWMEM_HELPER_ENABLE_DEBUG=false
#warn the user when memory is getting low 
GX_LOWMEM_HELPER_ENABLE_WARN=true
#whether to kill something or just warn, if set to true then warning will be overriden to true
GX_LOWMEM_HELPER_ENABLE_KILL=false

# kill firefox before anything else when the kernel invokes the out of memory killer
# recommended range: 0-1000
# chrome defaults to 300
GX_LOWMEM_HELPER_FF_OOM_ADJ=400
GX_LOWMEM_HELPER_FF_WC_OOM_ADJ=500
GX_LOWMEM_HELPER_FF_FP_OOM_ADJ=600

# how long to wait between checks
# recommended range: 10-120
GX_LOWMEM_HELPER_INTERVAL=60

#kill if total free ram+swap dips below this many mb, like oom killer but triggerred before things get out of hand
# recommended range: 100-500 depending on swap
# kernel oom killer will kick in when we get below 100
# if swappiness=10, warn/kill of 250/200 goes well for 256mb of swap, warn/kill of 500/250 goes well for 512mb of swap
GX_LOWMEM_HELPER_LOW_MEM_LIMIT_DEBUG=500
GX_LOWMEM_HELPER_LOW_MEM_LIMIT_WARN=500
GX_LOWMEM_HELPER_LOW_MEM_LIMIT_KILL=250

#kill if available ram dips below this in mb in combination with swap usage above this amount, use these if there is a huge amount of swap but you don't want to get too deep into it
# recommended range: 50-250
GX_LOWMEM_HELPER_LOW_RAM_LIMIT_DEBUG=100
GX_LOWMEM_HELPER_LOW_RAM_LIMIT_WARN=100
GX_LOWMEM_HELPER_LOW_RAM_LIMIT_KILL=100
# recommended range: 0-1000
# these need to change for different swappiness values
# maybe these should be percentages of ram - if ram is 2gb and swappiness=10, then 200mb is a good warning value, 
#  if 4gb of ram, warning might be able to be raised
GX_LOWMEM_HELPER_HIGH_SWAP_LIMIT_DEBUG=250
GX_LOWMEM_HELPER_HIGH_SWAP_LIMIT_WARN=250
GX_LOWMEM_HELPER_HIGH_SWAP_LIMIT_KILL=500

# catch runaway firefox flash plugin
GX_LOWMEM_HELPER_FFFP_RAM_LIMIT_DEBUG=30
GX_LOWMEM_HELPER_FFFP_RAM_LIMIT_WARN=40
GX_LOWMEM_HELPER_FFFP_RAM_LIMIT_KILL=50

#these variables are not used anymore
#if firefox and its plugin-container process use up more than this percentage of ram then kill ff
#GX_LOWMEM_HELPER_FF_MM_PERCENTAGE=90
#GX_LOWMEM_HELPER_FF_MM_PERCENTAGE_DT=75
##kill if high load in terms of load percentage, = 100 * load / #cpus ( not perfect metric ) and low ram
##once load is > 100, it can't keep up and can snowball really fast
## recommended range: 150-400
#GX_LOWMEM_HELPER_HIGH_LOAD_LIMIT=400
#GX_LOWMEM_HELPER_HIGH_LOAD_LIMIT_DT=200


#variable overrides
. /usr/share/groovix/global.conf

# if kill is turned on force warning on since we wont kill if not warned at least once
if [ $GX_LOWMEM_HELPER_ENABLE_KILL = true ] ; then
	GX_LOWMEM_HELPER_ENABLE_WARN=true
fi
WARNING_COUNT=0


if [ "$GX_LOWMEM_HELPER_ENABLE" != "true" ] ; then
	exit 0
fi

#initialize variables
DISABLE_WARNING_NOTIFICATION=false

LOGFILE=/var/log/groovix/system-debug.log

#this is no longer started from /etc/groovix/startup.d, start from /etc/groovix/session-setup.d so that we can do things like warn each user once

while true; do
        sleep $GX_LOWMEM_HELPER_INTERVAL
        DATETIME=`date +%Y-%m-%d-%H-%M-%S`

        #read FRAM FSWAP FTOTAL <<< $( free -mt |  awk '/Mem|Swap|Total/ {print $4}')
	read TRAM URAM FRAM ARAM TSWAP USWAP FSWAP TMEM UMEM FMEM <<< $( free -mt |  awk '/Mem|Swap|Total/ {print $2 "\n" $3 "\n" $4 "\n" $7}') ;
       
        # this is relatively expensive, lets disable this check for now
        #LOADP=$(awk '{printf("%i", $1/'$(nproc)' * 100 )}' /proc/loadavg)
	LOADP=0

	#PCPID "plugin-container" as listed by ps is both firefox content AND actual plugins like flash
	#PCPID=$(pidof plugin-container)
	FFPID=$(pidof firefox)		
	# firefox content , name as reported by top, not ps
	FFWCPID=$(pgrep  "Web Content")
	# firefox flash plugin - note the final r is cut off, as reported by top, not ps
	FFFPPID=$(pgrep "plugin-containe")

	# chrome already puts tabs at oom_score_adj = 300, set firefox to 400 and firefox plugin_container to 500
	for T in $FFPID ; do 
		echo $GX_LOWMEM_HELPER_FF_OOM_ADJ > /proc/$T/oom_score_adj
	done
	for T in $FFWCPID ; do 
		echo $GX_LOWMEM_HELPER_FF_WC_OOM_ADJ > /proc/$T/oom_score_adj
		
	done
	for T in $FFFPPID ; do 
		echo $GX_LOWMEM_HELPER_FF_FP_OOM_ADJ > /proc/$T/oom_score_adj
		
	done

	#the bigger memory usage as a percentage of ( firefox or plugin container  )
	# stuck pid of the current ps process in there too so we don't get nothing returned
	#FFMU=$(ps --no-headers -o '%mem' -p $$ $FFPID $FFWCPID $FFFPPID | cut -f 1 -d. | awk '{s+=$1} END {print s}')
        #special case - if firefox flash plugin memory usage > 50% of ram, kill it .  system gets too far into swap to recover if you wait for general out of memory killing
	FFFPMU=$(ps --no-headers -o '%mem' -p $$ $FFFPPID | cut -f 1 -d. | awk '{s+=$1} END {print s}')



	# make sure they have been warned at least once  
	if [[ $WARNING_COUNT -gt 0 &&  "$GX_LOWMEM_HELPER_ENABLE_KILL" = "true" && ( $FFFPMU -gt $GX_LOWMEM_HELPER_FFFP_RAM_LIMIT_KILL ||  $FMEM -lt $GX_LOWMEM_HELPER_LOW_MEM_LIMIT_KILL  || ( $ARAM -lt $GX_LOWMEM_HELPER_LOW_RAM_LIMIT_KILL  &&  $USWAP -gt $GX_LOWMEM_HELPER_HIGH_SWAP_LIMIT_KILL   ) ) ]] ; then
		#kill something if these are beyond the threshold values
		MEMHOG=$(for T in /proc/*/*score ; do cat $T | tr -d '\n' ;  echo $T | sed 's/\/proc\// /;s/\/oom_score/ /'  ; done | sort -n | tail -n 1 | awk '{ print $2 }')
        	MESSAGE="SEVERE: $DATETIME : system load = ${LOADP}% , avail ram = $ARAM , free mem total = $FMEM, used swap = $USWAP "
		ENDMESSAGE="killing $(cat /proc/$MEMHOG/comm)"
		#kill $MEMHOG 
		#sleep 2
		kill -9 $MEMHOG 

		# log after killing in case system is completely bogged down, but getting name of bad process is critical for debugging so do that before kill
		echo "$MESSAGE : $ENDMESSAGE" >> $LOGFILE

		ZENMESSAGE="Some applications had to be closed due to low available memory.\n\nPlease try to keep fewer applications/windows/tabs open."
		ZENTITLE="ERROR: Low System Memory"
		groovix-root-run-in-display zenity --error --title "$ZENTITLE" --text "$ZENMESSAGE"  &

	elif [[ $WARNING_COUNT -lt 1 && "$DISABLE_WARNING_NOTIFICATION" = "false" && "$GX_LOWMEM_HELPER_ENABLE_WARN" = "true" && ( $FFFPMU -gt $GX_LOWMEM_HELPER_FFFP_RAM_LIMIT_WARN ||   $FMEM -lt $GX_LOWMEM_HELPER_LOW_MEM_LIMIT_WARN  || ( $ARAM -lt $GX_LOWMEM_HELPER_LOW_RAM_LIMIT_WARN  &&  $USWAP -gt $GX_LOWMEM_HELPER_HIGH_SWAP_LIMIT_WARN   ) ) ]] ; then
		# instead of questions only give this warning 1 times, after that stop bugging them.  will only pop up  warning to user if they have closed the previous one
		WARNING_COUNT=$((WARNING_COUNT+1))
		# log warn user if these are beyond the threshold values
		#  after a few warnings stop bugging them, filling logs, turn on debugging if you want continuous info

		MESSAGE="WARNING: $DATETIME : system load = ${LOADP}% , avail ram = $ARAM , free mem total = $FMEM, used swap = $USWAP "
		ENDMESSAGE=" warning user " 
		echo "$MESSAGE : $ENDMESSAGE" >> $LOGFILE
		#additional debug info
		#free -mt >> $LOGFILE
		#cat /proc/loadavg >> $LOGFILE
		#top -n 1 -b  | head -n 20 >> $LOGFILE
		gxmemusage  | sort -n | tail >> $LOGFILE

		ZENTITLE="WARNING: Low System Memory"
		#ZENMESSAGE="Please try to keep fewer applications/windows/tabs open.\n\nDo you want to continue receiving this warning?"
		#groovix-root-run-in-display zenity --window-icon warning  --timeout 30  --question --title "$ZENTITLE" --text "$ZENMESSAGE" 
		#if [ $? -ne 0 ] ; then
		#	DISABLE_WARNING_NOTIFICATION=true
		#fi
		ZENMESSAGE="Please keep fewer applications/windows/tabs open or some applications may be forced to close."
		#ps aux | grep -v grep |  grep "WARNING: Low System Memory" ||  groovix-root-run-in-display zenity  --warning --title "$ZENTITLE" --text "$ZENMESSAGE" &
		groovix-root-run-in-display zenity  --warning --title "$ZENTITLE" --text "$ZENMESSAGE" &
		# give them some time to react to warning
		sleep $GX_LOWMEM_HELPER_INTERVAL


	elif [[ "$GX_LOWMEM_HELPER_ENABLE_KILL" != true &&  "$GX_LOWMEM_HELPER_ENABLE_DEBUG" = "true" && ( $FFFPMU -gt $GX_LOWMEM_HELPER_FFFP_RAM_LIMIT_DEBUG ||  $FMEM -lt $GX_LOWMEM_HELPER_LOW_MEM_LIMIT_DEBUG  || ( $ARAM -lt $GX_LOWMEM_HELPER_LOW_RAM_LIMIT_DEBUG  &&  $USWAP -gt $GX_LOWMEM_HELPER_HIGH_SWAP_LIMIT_DEBUG   ) ) ]] ; then
		#debug into log file if these are beyond the threshold values 
               	MESSAGE="DEBUG: $DATETIME : system load = ${LOADP}% , avail ram = $ARAM , free mem total = $FMEM, used swap = $USWAP "
		ENDMESSAGE=" debugging only " 
		echo "$MESSAGE : $ENDMESSAGE" >> $LOGFILE
	fi


done


