PeterThorpe: Created page with "= The following script is set to run every night via crontab = This get the usage of each user's home directory, puts it in a file. Then calls the python script to compare t..."

2018-10-23T14:12:30Z

Created page with "= The following script is set to run every night via crontab = This get the usage of each user's home directory, puts it in a file. Then calls the python script to compare t..."

New page

= The following script is set to run every night via crontab =

This get the usage of each user's home directory, puts it in a file. Then calls the python script to
compare this to the previous day.

The scripts are here: /mnt/system_usage/

Data is in : /mnt/system_usage/data

> more users_disk_usage.sh
#!/bin/bash
#####################################################################
# script to calculate the usage of all users.
# report how much disc space is used.
# compare users usage to yesterday.
# Over time this can be plotted and we can see changes and spikes
# Author: Peter Thorpe 20181013
#####################################################################
d=$(date +%Y_%m_%d)
yest=$(date --date="yesterday" +"%Y_%m_%d")
echo "$d"

cd /mnt/system_usage

# STEP: 1
# get the disk space info
df_cmd="df > ./data/current_disks_status_${d}.txt"
echo ${df_cmd}
eval ${df_cmd}

# command to calc and sort the usage of all the users:
cd /storage/home/users

# STEP: 2
# command to calc and sort the usage of all the users:
echo "calc: users usage."
# removed sort as this then requires du to have finished and
# takes too long | sort -n -rn
du_cmd="du -s * >
/mnt/system_usage/data/users_usage_${d}.txt"
echo ${du_cmd}
eval ${du_cmd}
wait

# STEP: 4
# python convert bytes to human readable and compare with yesterday:
cd /mnt/system_usage
echo "chewing with python."
py_cmd="/mnt/apps/Python3.7/bin/python3.7 compare_usage.py
--today /mnt/system_usage/data/users_usage_${d}.txt
--yesterday /mnt/system_usage/data/users_usage_${yest}.txt
-o /mnt/system_usage/data/USAGE_and_USAGE_CHANGE_${d}.txt"
echo ${py_cmd}
eval ${py_cmd}
wait

> more compare_usage.py
#####!/mnt/apps/Python3.7/bin python3.7
#####!/usr/bin/env python # this one is 2.6
####################################################################
# title: Compare the users usage form yesterday to today
# Script take in the bytes usage of the users on a linux system.
# generated by: du -s * | sort -n -rn > users_usage_${d}.txt
# The date is specified by bash: d=$(date +%Y_%m_%d).
# Yesterday is specified by bash too:
# yest=$(date --date="yesterday" +"%Y_%m_%d")
# It parses the two files and compare the change in usage, and reports
# human readable usage values.
# Author: Peter Thorpe 20181013
###################################################################

import os
import sys
import errno
import logging
import logging.handlers
import sys
from collections import defaultdict
import argparse
import datetime
import time

if sys.version_info[:2] != (3, 6):
# e.g. sys.version_info(major=3, minor=5, micro=2,
# releaselevel='final', serial=0)
# break the program
if sys.version_info[:2] != (3, 7):
print("currently using:", sys.version_info,
" version of python")
raise ImportError("Python 3.6 or 3.7 is required for ")
sys.exit(1)
VERSION = """compare usage v 0.001 \n"""
if "--version" in sys.argv:
print(VERSION)
sys.exit(1)

def get_args():
parser = argparse.ArgumentParser(description="parse usage data " +
"comper to yesterdays ",
add_help=False)
file_directory = os.path.realpath(__file__).split("compare_usage")[0]
optional = parser.add_argument_group('optional arguments')

optional.add_argument("-1", "--yesterday",
dest='yesterday',
action="store",
type=str,
default=None,
help="yesterday's usage file")

optional.add_argument("-2", "--today",
dest='today',
action="store",
default=None,
type=str,
help="todays usage file")

optional.add_argument("-o", "--out",
dest='out',
action="store",
default="usage_compared.out",
type=str,
help="todays usage file")

optional.add_argument("--logfile",
dest="logfile",
action="store",
default="usage_calculation.log",
type=str,
help="Logfile name")
args = parser.parse_args()
return args, file_directory

##########################################
# Functions

def test_line(line):
"""returns true lines. Not comments or blank line"""
if not line.strip():
return False # if the last line is blank
if line.startswith("#"):
return False # comment line
if line.startswith(" #"):
return False # comment line
return line

def sizeof_fmt(num, suffix='B'):
num = num * 1000
for unit in ['','K','M','G','T','P','E','Z']:
if abs(num) < 1024.0:
return "%3.1f%s%s" % (num, unit, suffix)
num /= 1024.0
return "%.1f%s%s" % (num, 'Yi', suffix)

def split_line(line):
"""func to split the output:
in look like 59703192 /storage/home/users/pjt6/
return usage and user and a list.
"""
usage_bytes, user = line.split()
return usage_bytes, user

def parse_file_with_dict(handle, dictionary):
"""func to populate the dic"""
for line in handle:
if test_line(line):
usage_bytes, user = split_line(line)
dictionary[user] = usage_bytes
return dictionary

def get_usage_change(today, yesterday):
"""func takes in the bytes of today and yesterdays usage
checks the biggest value and returns the change.
Positive or negatve"""
if today > yesterday:
change = "+"
usgae_change = today - yesterday
else:
usgae_change = yesterday - today
change = "-"
return usgae_change, change

FILECOUNT = 0
OLD_USAGE_DIC_COUNT = defaultdict(int)
NEW_USAGE_DIC_COUNT = defaultdict(int)
USAGE_CHANGE_DICT = defaultdict(str)

if __name__ == '__main__':
# Set up logging
args, FILE_DIRECTORY = get_args()
cwd = os.getcwd()
logger = logging.getLogger('compare_usage.py: %s' % time.asctime())
logger.setLevel(logging.DEBUG)
err_handler = logging.StreamHandler(sys.stderr)
err_formatter = logging.Formatter('%(levelname)s: %(message)s')
err_handler.setFormatter(err_formatter)
logger.addHandler(err_handler)
if args.logfile == "usage_calculation.log":
DATE_TIME = "%s" % (datetime.date.today())
DATE_TIME = DATE_TIME.replace("-", "_")
args.logfile = "%s_usage_calculation.log" % (DATE_TIME)
try:
logstream = open(args.logfile, 'w')
err_handler_file = logging.StreamHandler(logstream)
err_handler_file.setFormatter(err_formatter)
# logfile is always verbose
err_handler_file.setLevel(logging.INFO)
logger.addHandler(err_handler_file)
except:
outstr = "Could not open %s for logging" % args.logfile
logger.error(outstr)
sys.exit(1)
# Report input arguments
logger.info(sys.version_info)
logger.info("Command-line: %s", ' '.join(sys.argv))
logger.info("Starting testing: %s", time.asctime())
with open(args.yesterday) as handle:
FILECOUNT += 1

OLD_USAGE_DIC_COUNT = parse_file_with_dict(handle, OLD_USAGE_DIC_COUNT)
# print(OLD_USAGE_DIC_COUNT)

with open(args.today) as handle:
FILECOUNT += 1
NEW_USAGE_DIC_COUNT = parse_file_with_dict(handle, NEW_USAGE_DIC_COUNT)
# outfile
f_out = open(args.out, "w")
f_out.write("user\tusage_today\tusage_yesterday\tusage_change\n")
# compare the values in each dict for each user:
# dictionary[user] = usage_bytes
for user, usage_bytes in NEW_USAGE_DIC_COUNT.items():
usage_bytes = int(usage_bytes)
old_bytes = OLD_USAGE_DIC_COUNT[user]
old_bytes = int(old_bytes)
# fucntion should return a value then + or - depending on the usage change
usgae_change, change = get_usage_change(usage_bytes, old_bytes)
usgae_change_human = sizeof_fmt(usgae_change)
USAGE_CHANGE_DICT[user] = "%s%s" % (change, usgae_change_human)
outfmt = "%s\t%s\t%s\t%s\n" % (user, sizeof_fmt(usage_bytes),
sizeof_fmt(old_bytes),
usgae_change_human)
f_out.write(outfmt)
f_out.close()

Users disk usage - Revision history

PeterThorpe: Created page with "= The following script is set to run every night via crontab = This get the usage of each user's home directory, puts it in a file. Then calls the python script to compare t..."